From 4a15e02dced37a637e115a85b3158f05cf5d6871 Mon Sep 17 00:00:00 2001 From: root Date: Wed, 1 Feb 2023 02:38:14 +0000 Subject: [PATCH 01/33] Add pin_min_versions_to_ci_deps() --- scripts/validate_min_versions_in_sync.py | 64 ++++++++++++++++++++++++ 1 file changed, 64 insertions(+) diff --git a/scripts/validate_min_versions_in_sync.py b/scripts/validate_min_versions_in_sync.py index 7c102096c1690..5a69553aa4596 100755 --- a/scripts/validate_min_versions_in_sync.py +++ b/scripts/validate_min_versions_in_sync.py @@ -17,6 +17,8 @@ import pathlib import sys +import yaml + if sys.version_info >= (3, 11): import tomllib else: @@ -28,6 +30,7 @@ ) CODE_PATH = pathlib.Path("pandas/compat/_optional.py").resolve() SETUP_PATH = pathlib.Path("pyproject.toml").resolve() +YAML_PATH = pathlib.Path("ci/deps") EXCLUDE_DEPS = {"tzdata", "blosc"} # pandas package is not available # in pre-commit environment @@ -41,6 +44,57 @@ import _optional +def pin_min_versions_to_ci_deps(): + toml_dependencies = get_versions_from_toml() + print("TOML Dependencies", toml_dependencies) + # if toml package is in yaml file, then run condition check + # TODO: Add environment.yml to "all_yaml_files" + all_yaml_files = list(YAML_PATH.iterdir()) + yaml_dependencies = {} + # iterate through each yaml file + for curr_file in all_yaml_files: + with open(curr_file, "rb") as yaml_f: + # all stuff including dependencies + yaml_file = yaml.safe_load(yaml_f) + print("\nCurrent YAML File Path:", curr_file) + # just yaml deps + yaml_deps = yaml_file["dependencies"] + print("YAML Dictionary:", yaml_deps) + # iterate through package/version dependency string + for dependency in yaml_deps: + # clean the string. extract dictionary data: yaml package, version + if ">=" in dependency: + yaml_package, yaml_version = str(dependency).strip().split(">=") + elif "=" in dependency: + package, version = str(dependency).strip().split("=") + else: + package, version = str(dependency), "None" + # comparison between YAML/TOML + if ">=" in dependency or "=" in dependency: + if yaml_package in toml_dependencies: + if toml_dependencies[yaml_package] > yaml_version: + # update yaml package version to toml min version + pass + else: + if yaml_package in toml_dependencies: + # update yaml package version to toml min version + # use ">=" operator + pass + + # put extracted data into yaml dictionary + yaml_dependencies[package] = version + print() + myKeys = list(yaml_dependencies.keys()) + myKeys.sort() + sorted_dict = {i: yaml_dependencies[i] for i in myKeys} + + print("Sorted YAML Dependencies:") + for item in sorted_dict.items(): + print(item) + print("===================") + return yaml_dependencies + + def get_versions_from_code() -> dict[str, str]: """Min versions for checking within pandas code.""" install_map = _optional.INSTALL_MAPPING @@ -108,11 +162,21 @@ def get_versions_from_toml() -> dict[str, str]: for item in EXCLUDE_DEPS: optional_dependencies.pop(item, None) + print() + myKeys = list(optional_dependencies.keys()) + myKeys.sort() + sorted_dict = {i: optional_dependencies[i] for i in myKeys} + print("Sorted TOML Dependencies:") + for item in sorted_dict.items(): + print(item) return optional_dependencies def main(): + yaml_dependencies = pin_min_versions_to_ci_deps() + print("YAML Dependencies:", yaml_dependencies) + with open(CI_PATH, encoding="utf-8") as f: _, ci_optional = get_versions_from_ci(f.readlines()) code_optional = get_versions_from_code() From f71a8696e6b37b523f8473218a8fd0c8989c9fea Mon Sep 17 00:00:00 2001 From: root Date: Wed, 1 Feb 2023 03:08:27 +0000 Subject: [PATCH 02/33] Fix typos --- scripts/validate_min_versions_in_sync.py | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/scripts/validate_min_versions_in_sync.py b/scripts/validate_min_versions_in_sync.py index 5a69553aa4596..ac0541e605027 100755 --- a/scripts/validate_min_versions_in_sync.py +++ b/scripts/validate_min_versions_in_sync.py @@ -66,15 +66,16 @@ def pin_min_versions_to_ci_deps(): if ">=" in dependency: yaml_package, yaml_version = str(dependency).strip().split(">=") elif "=" in dependency: - package, version = str(dependency).strip().split("=") + yaml_package, yaml_version = str(dependency).strip().split("=") else: - package, version = str(dependency), "None" + yaml_package, yaml_version = str(dependency), "None" # comparison between YAML/TOML - if ">=" in dependency or "=" in dependency: + if "=" in dependency: if yaml_package in toml_dependencies: if toml_dependencies[yaml_package] > yaml_version: # update yaml package version to toml min version pass + # else - yaml package has no version pinned else: if yaml_package in toml_dependencies: # update yaml package version to toml min version @@ -82,7 +83,7 @@ def pin_min_versions_to_ci_deps(): pass # put extracted data into yaml dictionary - yaml_dependencies[package] = version + yaml_dependencies[yaml_package] = yaml_version print() myKeys = list(yaml_dependencies.keys()) myKeys.sort() From e4b90a22eafa8fc1481c22337595ee7cfcf06028 Mon Sep 17 00:00:00 2001 From: root Date: Wed, 1 Feb 2023 03:45:29 +0000 Subject: [PATCH 03/33] Remove debug statements --- scripts/validate_min_versions_in_sync.py | 40 ++---------------------- 1 file changed, 2 insertions(+), 38 deletions(-) diff --git a/scripts/validate_min_versions_in_sync.py b/scripts/validate_min_versions_in_sync.py index ac0541e605027..17ec0b66a71d6 100755 --- a/scripts/validate_min_versions_in_sync.py +++ b/scripts/validate_min_versions_in_sync.py @@ -46,55 +46,29 @@ def pin_min_versions_to_ci_deps(): toml_dependencies = get_versions_from_toml() - print("TOML Dependencies", toml_dependencies) - # if toml package is in yaml file, then run condition check # TODO: Add environment.yml to "all_yaml_files" all_yaml_files = list(YAML_PATH.iterdir()) - yaml_dependencies = {} - # iterate through each yaml file for curr_file in all_yaml_files: with open(curr_file, "rb") as yaml_f: - # all stuff including dependencies yaml_file = yaml.safe_load(yaml_f) - print("\nCurrent YAML File Path:", curr_file) - # just yaml deps yaml_deps = yaml_file["dependencies"] - print("YAML Dictionary:", yaml_deps) - # iterate through package/version dependency string for dependency in yaml_deps: - # clean the string. extract dictionary data: yaml package, version if ">=" in dependency: yaml_package, yaml_version = str(dependency).strip().split(">=") elif "=" in dependency: yaml_package, yaml_version = str(dependency).strip().split("=") else: yaml_package, yaml_version = str(dependency), "None" - # comparison between YAML/TOML if "=" in dependency: if yaml_package in toml_dependencies: if toml_dependencies[yaml_package] > yaml_version: # update yaml package version to toml min version pass - # else - yaml package has no version pinned else: if yaml_package in toml_dependencies: - # update yaml package version to toml min version - # use ">=" operator + # update yaml package version to toml min version. use ">=" pass - # put extracted data into yaml dictionary - yaml_dependencies[yaml_package] = yaml_version - print() - myKeys = list(yaml_dependencies.keys()) - myKeys.sort() - sorted_dict = {i: yaml_dependencies[i] for i in myKeys} - - print("Sorted YAML Dependencies:") - for item in sorted_dict.items(): - print(item) - print("===================") - return yaml_dependencies - def get_versions_from_code() -> dict[str, str]: """Min versions for checking within pandas code.""" @@ -163,21 +137,11 @@ def get_versions_from_toml() -> dict[str, str]: for item in EXCLUDE_DEPS: optional_dependencies.pop(item, None) - print() - myKeys = list(optional_dependencies.keys()) - myKeys.sort() - sorted_dict = {i: optional_dependencies[i] for i in myKeys} - print("Sorted TOML Dependencies:") - for item in sorted_dict.items(): - print(item) - return optional_dependencies def main(): - yaml_dependencies = pin_min_versions_to_ci_deps() - print("YAML Dependencies:", yaml_dependencies) - + pin_min_versions_to_ci_deps() with open(CI_PATH, encoding="utf-8") as f: _, ci_optional = get_versions_from_ci(f.readlines()) code_optional = get_versions_from_code() From 527bd37caf8a9747b49a126a1d95249752508f65 Mon Sep 17 00:00:00 2001 From: doge Date: Wed, 1 Feb 2023 19:00:12 -0800 Subject: [PATCH 04/33] Update YAML package versions --- scripts/validate_min_versions_in_sync.py | 49 ++++++++++++++++-------- 1 file changed, 33 insertions(+), 16 deletions(-) diff --git a/scripts/validate_min_versions_in_sync.py b/scripts/validate_min_versions_in_sync.py index 17ec0b66a71d6..c2869e2d9d183 100755 --- a/scripts/validate_min_versions_in_sync.py +++ b/scripts/validate_min_versions_in_sync.py @@ -31,6 +31,7 @@ CODE_PATH = pathlib.Path("pandas/compat/_optional.py").resolve() SETUP_PATH = pathlib.Path("pyproject.toml").resolve() YAML_PATH = pathlib.Path("ci/deps") +ENV_PATH = pathlib.Path("environment.yml") EXCLUDE_DEPS = {"tzdata", "blosc"} # pandas package is not available # in pre-commit environment @@ -46,28 +47,44 @@ def pin_min_versions_to_ci_deps(): toml_dependencies = get_versions_from_toml() - # TODO: Add environment.yml to "all_yaml_files" - all_yaml_files = list(YAML_PATH.iterdir()) + all_yaml_files = list(YAML_PATH.iterdir()).append(ENV_PATH) + for curr_file in all_yaml_files: with open(curr_file, "rb") as yaml_f: + data = yaml_f.read() yaml_file = yaml.safe_load(yaml_f) yaml_deps = yaml_file["dependencies"] - for dependency in yaml_deps: - if ">=" in dependency: - yaml_package, yaml_version = str(dependency).strip().split(">=") - elif "=" in dependency: - yaml_package, yaml_version = str(dependency).strip().split("=") + yaml_left = "" + for dependency_line in yaml_deps: + if ">=" in dependency_line: + yaml_package, yaml_version = ( + str(dependency_line).strip().split(">=") + ) + yaml_left = yaml_package + ">=" + elif "=" in dependency_line: + yaml_package, yaml_version = str(dependency_line).strip().split("=") + yaml_left = yaml_package + "=" else: - yaml_package, yaml_version = str(dependency), "None" - if "=" in dependency: - if yaml_package in toml_dependencies: + yaml_package, yaml_version = str(dependency_line).strip(), None + yaml_package = yaml_package[2:] + if yaml_package in toml_dependencies: + # update yaml package version to TOML min version + if yaml_version is not None: if toml_dependencies[yaml_package] > yaml_version: - # update yaml package version to toml min version - pass - else: - if yaml_package in toml_dependencies: - # update yaml package version to toml min version. use ">=" - pass + # ex: "hypothesis>=" + "6.34.2" + replace_text = yaml_left + toml_dependencies[yaml_package] + else: + # ex: "hypothesis + ">=" + 6.34.2" + replace_text = ( + yaml_left + ">=" + toml_dependencies[yaml_package] + ) + search_text = str(dependency_line).strip()[2:] + data = data.replace(search_text, replace_text) + + # create new yaml file with updated versions + with open(curr_file, "w") as f: + f.write(data) + # TODO: delete old yaml file def get_versions_from_code() -> dict[str, str]: From cbd87e5db94b1c6df36c5646ccc3c30bf039e535 Mon Sep 17 00:00:00 2001 From: doge Date: Thu, 2 Feb 2023 17:14:36 -0800 Subject: [PATCH 05/33] Replace files with updated min versions --- ci/deps/actions-pypy-38.yaml | 2 +- scripts/validate_min_versions_in_sync.py | 39 +++++++++++++----------- 2 files changed, 22 insertions(+), 19 deletions(-) diff --git a/ci/deps/actions-pypy-38.yaml b/ci/deps/actions-pypy-38.yaml index 3218ec13a9c40..5efb39e4116b2 100644 --- a/ci/deps/actions-pypy-38.yaml +++ b/ci/deps/actions-pypy-38.yaml @@ -5,7 +5,7 @@ dependencies: # TODO: Add the rest of the dependencies in here # once the other plentiful failures/segfaults # with base pandas has been dealt with - - python=3.8[build=*_pypy] # TODO: use this once pypy3.8 is available + - python=3.8 #[build=*_pypy] # TODO: use this once pypy3.8 is available # build dependencies - versioneer[toml] diff --git a/scripts/validate_min_versions_in_sync.py b/scripts/validate_min_versions_in_sync.py index c2869e2d9d183..a3fc7da120624 100755 --- a/scripts/validate_min_versions_in_sync.py +++ b/scripts/validate_min_versions_in_sync.py @@ -14,6 +14,7 @@ """ from __future__ import annotations +import os import pathlib import sys @@ -47,44 +48,46 @@ def pin_min_versions_to_ci_deps(): toml_dependencies = get_versions_from_toml() - all_yaml_files = list(YAML_PATH.iterdir()).append(ENV_PATH) + all_yaml_files = list(YAML_PATH.iterdir()) + all_yaml_files.append(ENV_PATH) for curr_file in all_yaml_files: - with open(curr_file, "rb") as yaml_f: + with open(curr_file) as yaml_f: data = yaml_f.read() - yaml_file = yaml.safe_load(yaml_f) + yaml_file = yaml.safe_load(data) yaml_deps = yaml_file["dependencies"] yaml_left = "" - for dependency_line in yaml_deps: - if ">=" in dependency_line: - yaml_package, yaml_version = ( - str(dependency_line).strip().split(">=") - ) + for dep_line in yaml_deps: + replace_text = "" + if ">=" in dep_line: + yaml_package, yaml_version = str(dep_line).strip().split(">=") yaml_left = yaml_package + ">=" - elif "=" in dependency_line: - yaml_package, yaml_version = str(dependency_line).strip().split("=") + elif "=" in dep_line: + yaml_package, yaml_version = str(dep_line).strip().split("=") yaml_left = yaml_package + "=" else: - yaml_package, yaml_version = str(dependency_line).strip(), None - yaml_package = yaml_package[2:] + yaml_package, yaml_version = str(dep_line).strip(), None if yaml_package in toml_dependencies: # update yaml package version to TOML min version if yaml_version is not None: if toml_dependencies[yaml_package] > yaml_version: # ex: "hypothesis>=" + "6.34.2" replace_text = yaml_left + toml_dependencies[yaml_package] + search_text = str(dep_line) + data = data.replace(search_text, replace_text) else: # ex: "hypothesis + ">=" + 6.34.2" replace_text = ( - yaml_left + ">=" + toml_dependencies[yaml_package] + yaml_package + ">=" + toml_dependencies[yaml_package] ) - search_text = str(dependency_line).strip()[2:] - data = data.replace(search_text, replace_text) - - # create new yaml file with updated versions + search_text = str(dep_line) + data = data.replace(search_text, replace_text) + else: + pass + replace_text = "" + os.remove(curr_file) with open(curr_file, "w") as f: f.write(data) - # TODO: delete old yaml file def get_versions_from_code() -> dict[str, str]: From c0c4ffce6230dfb77cd2b0a3f86baa45c755e317 Mon Sep 17 00:00:00 2001 From: doge Date: Sun, 5 Feb 2023 18:49:51 -0800 Subject: [PATCH 06/33] Add exclusions + edge cases --- scripts/validate_min_versions_in_sync.py | 61 +++++++++++++++++------- 1 file changed, 44 insertions(+), 17 deletions(-) diff --git a/scripts/validate_min_versions_in_sync.py b/scripts/validate_min_versions_in_sync.py index a3fc7da120624..dfcfae47e8bad 100755 --- a/scripts/validate_min_versions_in_sync.py +++ b/scripts/validate_min_versions_in_sync.py @@ -47,44 +47,71 @@ def pin_min_versions_to_ci_deps(): + exclusion_list = { + "python=3.8[build=*_pypy]": None, + "actions-38-minimum_versions.yaml": None, + } toml_dependencies = get_versions_from_toml() all_yaml_files = list(YAML_PATH.iterdir()) all_yaml_files.append(ENV_PATH) - for curr_file in all_yaml_files: with open(curr_file) as yaml_f: data = yaml_f.read() yaml_file = yaml.safe_load(data) yaml_deps = yaml_file["dependencies"] - yaml_left = "" for dep_line in yaml_deps: - replace_text = "" - if ">=" in dep_line: - yaml_package, yaml_version = str(dep_line).strip().split(">=") - yaml_left = yaml_package + ">=" - elif "=" in dep_line: - yaml_package, yaml_version = str(dep_line).strip().split("=") - yaml_left = yaml_package + "=" + replace_text = operator = yaml_left = "" + search_text = str(dep_line) + if ( + str(dep_line) in exclusion_list + or str(curr_file)[8:] in exclusion_list + ): + continue + if "=" in dep_line: + operator = "=" + elif "==" in dep_line: + operator = "==" + elif ">=" in dep_line: + operator = ">=" + elif "<" in dep_line: + operator = "<" + elif ">" in dep_line: + operator = ">" else: + operator = "" + if operator == "": yaml_package, yaml_version = str(dep_line).strip(), None + yaml_left = yaml_package + else: + yaml_package, yaml_version = str(dep_line).strip().split(operator) + if operator == "<" or operator == ">": + if yaml_package in toml_dependencies: + if yaml_version <= toml_dependencies[yaml_package]: + yaml_left = yaml_package + else: + yaml_left = str(dep_line) + ", " + else: + yaml_left = yaml_package + operator if yaml_package in toml_dependencies: + if "=" not in yaml_left and ">" in yaml_left or "<" in yaml_left: + if "," in yaml_left: + # ex: "numpy<1.24.0," + ">=" + "1.2" + replace_text = ( + yaml_left + ">=" + toml_dependencies[yaml_package] + ) + else: + replace_text = yaml_left + toml_dependencies[yaml_package] # update yaml package version to TOML min version - if yaml_version is not None: + elif yaml_version is not None: if toml_dependencies[yaml_package] > yaml_version: # ex: "hypothesis>=" + "6.34.2" replace_text = yaml_left + toml_dependencies[yaml_package] - search_text = str(dep_line) - data = data.replace(search_text, replace_text) else: # ex: "hypothesis + ">=" + 6.34.2" replace_text = ( yaml_package + ">=" + toml_dependencies[yaml_package] ) - search_text = str(dep_line) - data = data.replace(search_text, replace_text) - else: - pass - replace_text = "" + data = data.replace(search_text, replace_text) os.remove(curr_file) with open(curr_file, "w") as f: f.write(data) From e4789d21c8e82570f94f1fc4e525f22eab99d275 Mon Sep 17 00:00:00 2001 From: doge Date: Sun, 5 Feb 2023 19:08:18 -0800 Subject: [PATCH 07/33] Revert file to original --- ci/deps/actions-pypy-38.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ci/deps/actions-pypy-38.yaml b/ci/deps/actions-pypy-38.yaml index 5efb39e4116b2..3218ec13a9c40 100644 --- a/ci/deps/actions-pypy-38.yaml +++ b/ci/deps/actions-pypy-38.yaml @@ -5,7 +5,7 @@ dependencies: # TODO: Add the rest of the dependencies in here # once the other plentiful failures/segfaults # with base pandas has been dealt with - - python=3.8 #[build=*_pypy] # TODO: use this once pypy3.8 is available + - python=3.8[build=*_pypy] # TODO: use this once pypy3.8 is available # build dependencies - versioneer[toml] From 1c8c8b48ce3d5629c0370f7480914a951c06fb7c Mon Sep 17 00:00:00 2001 From: doge Date: Tue, 7 Feb 2023 13:50:10 -0800 Subject: [PATCH 08/33] Revert environment.yml --- environment.yml | 16 ++++++++-------- requirements-dev.txt | 4 ++-- scripts/validate_min_versions_in_sync.py | 3 ++- 3 files changed, 12 insertions(+), 11 deletions(-) diff --git a/environment.yml b/environment.yml index 47627fcac32e1..1fd9050bdd6a3 100644 --- a/environment.yml +++ b/environment.yml @@ -38,12 +38,12 @@ dependencies: - lxml - matplotlib>=3.6.1 - numba>=0.53.1 - - numexpr>=2.8.0 # pin for "Run checks on imported code" job + - numexpr>=2.8.0 # pin for "Run checks on imported code" job - openpyxl - odfpy - py - psycopg2 - - pyarrow + - pyarrow<11 - pymysql - pyreadstat - pytables @@ -78,19 +78,19 @@ dependencies: - black=22.10.0 - cpplint - flake8=6.0.0 - - isort>=5.2.1 # check that imports are in the right order - - mypy=0.991 + - isort>=5.2.1 # check that imports are in the right order + - mypy=1.0 - pre-commit>=2.15.0 - pyupgrade - ruff=0.0.215 # documentation - - gitpython # obtain contributors from git for whatsnew + - gitpython # obtain contributors from git for whatsnew - gitdb - - natsort # DataFrame.sort_values doctest + - natsort # DataFrame.sort_values doctest - numpydoc - pydata-sphinx-theme<0.11 - - pytest-cython # doctest + - pytest-cython # doctest - sphinx - sphinx-panels - sphinx-copybutton @@ -109,7 +109,7 @@ dependencies: - ipykernel # web - - jinja2 # in optional dependencies, but documented here as needed + - jinja2 # in optional dependencies, but documented here as needed - markdown - feedparser - pyyaml diff --git a/requirements-dev.txt b/requirements-dev.txt index 112f90222427b..3783c7c2aeb5f 100644 --- a/requirements-dev.txt +++ b/requirements-dev.txt @@ -32,7 +32,7 @@ openpyxl odfpy py psycopg2-binary -pyarrow +pyarrow<11 pymysql pyreadstat tables @@ -56,7 +56,7 @@ black==22.10.0 cpplint flake8==6.0.0 isort>=5.2.1 -mypy==0.991 +mypy==1.0 pre-commit>=2.15.0 pyupgrade ruff==0.0.215 diff --git a/scripts/validate_min_versions_in_sync.py b/scripts/validate_min_versions_in_sync.py index dfcfae47e8bad..819f1c4d26121 100755 --- a/scripts/validate_min_versions_in_sync.py +++ b/scripts/validate_min_versions_in_sync.py @@ -49,11 +49,12 @@ def pin_min_versions_to_ci_deps(): exclusion_list = { "python=3.8[build=*_pypy]": None, - "actions-38-minimum_versions.yaml": None, + # "actions-38-minimum_versions.yaml": None, } toml_dependencies = get_versions_from_toml() all_yaml_files = list(YAML_PATH.iterdir()) all_yaml_files.append(ENV_PATH) + print("BLAH", all_yaml_files) for curr_file in all_yaml_files: with open(curr_file) as yaml_f: data = yaml_f.read() From 6d6c391540cc243e1a19e8894c4517a95f6df7f7 Mon Sep 17 00:00:00 2001 From: doge Date: Wed, 8 Feb 2023 16:40:06 -0800 Subject: [PATCH 09/33] Fix bugs --- scripts/validate_min_versions_in_sync.py | 32 ++++++++++++++---------- 1 file changed, 19 insertions(+), 13 deletions(-) diff --git a/scripts/validate_min_versions_in_sync.py b/scripts/validate_min_versions_in_sync.py index 819f1c4d26121..f4586c3fc5e19 100755 --- a/scripts/validate_min_versions_in_sync.py +++ b/scripts/validate_min_versions_in_sync.py @@ -49,31 +49,29 @@ def pin_min_versions_to_ci_deps(): exclusion_list = { "python=3.8[build=*_pypy]": None, - # "actions-38-minimum_versions.yaml": None, } toml_dependencies = get_versions_from_toml() all_yaml_files = list(YAML_PATH.iterdir()) all_yaml_files.append(ENV_PATH) - print("BLAH", all_yaml_files) for curr_file in all_yaml_files: with open(curr_file) as yaml_f: data = yaml_f.read() yaml_file = yaml.safe_load(data) yaml_deps = yaml_file["dependencies"] + res = [] + [res.append(x) for x in yaml_deps if x not in res] + yaml_deps = res for dep_line in yaml_deps: replace_text = operator = yaml_left = "" search_text = str(dep_line) - if ( - str(dep_line) in exclusion_list - or str(curr_file)[8:] in exclusion_list - ): + if str(dep_line) in exclusion_list: continue - if "=" in dep_line: - operator = "=" + if ">=" in dep_line: + operator = ">=" elif "==" in dep_line: operator = "==" - elif ">=" in dep_line: - operator = ">=" + elif "=" in dep_line: + operator = "=" elif "<" in dep_line: operator = "<" elif ">" in dep_line: @@ -87,14 +85,16 @@ def pin_min_versions_to_ci_deps(): yaml_package, yaml_version = str(dep_line).strip().split(operator) if operator == "<" or operator == ">": if yaml_package in toml_dependencies: - if yaml_version <= toml_dependencies[yaml_package]: + if version.parse(yaml_version) <= version.parse( + toml_dependencies[yaml_package] + ): yaml_left = yaml_package else: yaml_left = str(dep_line) + ", " else: yaml_left = yaml_package + operator if yaml_package in toml_dependencies: - if "=" not in yaml_left and ">" in yaml_left or "<" in yaml_left: + if ">" in yaml_left or "<" in yaml_left: if "," in yaml_left: # ex: "numpy<1.24.0," + ">=" + "1.2" replace_text = ( @@ -104,9 +104,15 @@ def pin_min_versions_to_ci_deps(): replace_text = yaml_left + toml_dependencies[yaml_package] # update yaml package version to TOML min version elif yaml_version is not None: - if toml_dependencies[yaml_package] > yaml_version: + if version.parse( + toml_dependencies[yaml_package] + ) > version.parse(yaml_version): # ex: "hypothesis>=" + "6.34.2" replace_text = yaml_left + toml_dependencies[yaml_package] + elif version.parse( + toml_dependencies[yaml_package] + ) == version.parse(yaml_version): + replace_text = dep_line else: # ex: "hypothesis + ">=" + 6.34.2" replace_text = ( From a4b4bdb01c0ccd30f2a1408d798bf35f81ef4a13 Mon Sep 17 00:00:00 2001 From: doge Date: Thu, 9 Feb 2023 14:45:45 -0800 Subject: [PATCH 10/33] Add helper function --- .pre-commit-config.yaml | 2 +- environment.yml | 14 +-- requirements-dev.txt | 2 +- scripts/validate_min_versions_in_sync.py | 137 ++++++++++++----------- 4 files changed, 78 insertions(+), 77 deletions(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index c5c5cc09a0e7a..1c8ae99fe21f2 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -397,7 +397,7 @@ repos: entry: python scripts/validate_min_versions_in_sync.py language: python files: ^(ci/deps/actions-.*-minimum_versions\.yaml|pandas/compat/_optional\.py)$ - additional_dependencies: [tomli] + additional_dependencies: [tomli, pyyaml] - id: validate-errors-locations name: Validate errors locations description: Validate errors are in appropriate locations. diff --git a/environment.yml b/environment.yml index 1fd9050bdd6a3..9169cbf08b45d 100644 --- a/environment.yml +++ b/environment.yml @@ -38,7 +38,7 @@ dependencies: - lxml - matplotlib>=3.6.1 - numba>=0.53.1 - - numexpr>=2.8.0 # pin for "Run checks on imported code" job + - numexpr>=2.8.0 # pin for "Run checks on imported code" job - openpyxl - odfpy - py @@ -51,7 +51,7 @@ dependencies: - pyxlsb - s3fs>=2021.08.0 - scipy - - sqlalchemy<1.4.46 + - sqlalchemy - tabulate - tzdata>=2022a - xarray @@ -78,19 +78,19 @@ dependencies: - black=22.10.0 - cpplint - flake8=6.0.0 - - isort>=5.2.1 # check that imports are in the right order + - isort>=5.2.1 # check that imports are in the right order - mypy=1.0 - pre-commit>=2.15.0 - pyupgrade - ruff=0.0.215 # documentation - - gitpython # obtain contributors from git for whatsnew + - gitpython # obtain contributors from git for whatsnew - gitdb - - natsort # DataFrame.sort_values doctest + - natsort # DataFrame.sort_values doctest - numpydoc - pydata-sphinx-theme<0.11 - - pytest-cython # doctest + - pytest-cython # doctest - sphinx - sphinx-panels - sphinx-copybutton @@ -109,7 +109,7 @@ dependencies: - ipykernel # web - - jinja2 # in optional dependencies, but documented here as needed + - jinja2 # in optional dependencies, but documented here as needed - markdown - feedparser - pyyaml diff --git a/requirements-dev.txt b/requirements-dev.txt index 3783c7c2aeb5f..b6992a7266600 100644 --- a/requirements-dev.txt +++ b/requirements-dev.txt @@ -40,7 +40,7 @@ python-snappy pyxlsb s3fs>=2021.08.0 scipy -sqlalchemy<1.4.46 +sqlalchemy tabulate tzdata>=2022.1 xarray diff --git a/scripts/validate_min_versions_in_sync.py b/scripts/validate_min_versions_in_sync.py index f4586c3fc5e19..9835cf6cf513b 100755 --- a/scripts/validate_min_versions_in_sync.py +++ b/scripts/validate_min_versions_in_sync.py @@ -47,83 +47,84 @@ def pin_min_versions_to_ci_deps(): - exclusion_list = { - "python=3.8[build=*_pypy]": None, - } - toml_dependencies = get_versions_from_toml() all_yaml_files = list(YAML_PATH.iterdir()) all_yaml_files.append(ENV_PATH) for curr_file in all_yaml_files: with open(curr_file) as yaml_f: - data = yaml_f.read() - yaml_file = yaml.safe_load(data) - yaml_deps = yaml_file["dependencies"] - res = [] - [res.append(x) for x in yaml_deps if x not in res] - yaml_deps = res - for dep_line in yaml_deps: - replace_text = operator = yaml_left = "" - search_text = str(dep_line) - if str(dep_line) in exclusion_list: - continue - if ">=" in dep_line: - operator = ">=" - elif "==" in dep_line: - operator = "==" - elif "=" in dep_line: - operator = "=" - elif "<" in dep_line: - operator = "<" - elif ">" in dep_line: - operator = ">" - else: - operator = "" - if operator == "": - yaml_package, yaml_version = str(dep_line).strip(), None - yaml_left = yaml_package - else: - yaml_package, yaml_version = str(dep_line).strip().split(operator) - if operator == "<" or operator == ">": - if yaml_package in toml_dependencies: - if version.parse(yaml_version) <= version.parse( - toml_dependencies[yaml_package] - ): - yaml_left = yaml_package - else: - yaml_left = str(dep_line) + ", " - else: - yaml_left = yaml_package + operator - if yaml_package in toml_dependencies: - if ">" in yaml_left or "<" in yaml_left: - if "," in yaml_left: - # ex: "numpy<1.24.0," + ">=" + "1.2" - replace_text = ( - yaml_left + ">=" + toml_dependencies[yaml_package] - ) - else: - replace_text = yaml_left + toml_dependencies[yaml_package] - # update yaml package version to TOML min version - elif yaml_version is not None: - if version.parse( - toml_dependencies[yaml_package] - ) > version.parse(yaml_version): - # ex: "hypothesis>=" + "6.34.2" - replace_text = yaml_left + toml_dependencies[yaml_package] - elif version.parse( - toml_dependencies[yaml_package] - ) == version.parse(yaml_version): - replace_text = dep_line - else: - # ex: "hypothesis + ">=" + 6.34.2" - replace_text = ( - yaml_package + ">=" + toml_dependencies[yaml_package] - ) - data = data.replace(search_text, replace_text) + data = update_yaml_file_version(yaml_f) os.remove(curr_file) with open(curr_file, "w") as f: f.write(data) +def update_yaml_file_version(yaml_file): + exclusion_list = { + "python=3.8[build=*_pypy]": None, + } + toml_dependencies = get_versions_from_toml() + data = yaml_file.read() + yaml_f = yaml.safe_load(data) + yaml_deps = yaml_f["dependencies"] + res = [] + [res.append(x) for x in yaml_deps if x not in res] + yaml_deps = res + for dep_line in yaml_deps: + replace_text = operator = yaml_left = "" + search_text = str(dep_line) + if str(dep_line) in exclusion_list: + continue + if ">=" in dep_line: + operator = ">=" + elif "==" in dep_line: + operator = "==" + elif "=" in dep_line: + operator = "=" + elif "<" in dep_line: + operator = "<" + elif ">" in dep_line: + operator = ">" + else: + operator = "" + if operator == "": + yaml_package, yaml_version = str(dep_line).strip(), None + yaml_left = yaml_package + else: + yaml_package, yaml_version = str(dep_line).strip().split(operator) + if operator == "<" or operator == ">": + if yaml_package in toml_dependencies: + if version.parse(yaml_version) <= version.parse( + toml_dependencies[yaml_package] + ): + yaml_left = yaml_package + else: + yaml_left = str(dep_line) + ", " + else: + yaml_left = yaml_package + operator + if yaml_package in toml_dependencies: + if ">" in yaml_left or "<" in yaml_left: + if "," in yaml_left: + # ex: "numpy<1.24.0," + ">=" + "1.2" + replace_text = yaml_left + ">=" + toml_dependencies[yaml_package] + else: + replace_text = yaml_left + toml_dependencies[yaml_package] + # update yaml package version to TOML min version + elif yaml_version is not None: + if version.parse(toml_dependencies[yaml_package]) > version.parse( + yaml_version + ): + # ex: "hypothesis>=" + "6.34.2" + replace_text = yaml_left + toml_dependencies[yaml_package] + elif version.parse(toml_dependencies[yaml_package]) == version.parse( + yaml_version + ): + replace_text = dep_line + else: + # ex: "hypothesis + ">=" + 6.34.2" + replace_text = yaml_package + ">=" + toml_dependencies[yaml_package] + data = data.replace(search_text, replace_text) + return data + + def get_versions_from_code() -> dict[str, str]: """Min versions for checking within pandas code.""" install_map = _optional.INSTALL_MAPPING From e2bb536a495f8e4f21098145f1756cfe2b69b3fc Mon Sep 17 00:00:00 2001 From: doge Date: Thu, 9 Feb 2023 14:49:21 -0800 Subject: [PATCH 11/33] Ran pre-commit run validate-min-versions-in-sync --all-files --- ci/deps/actions-310.yaml | 58 +++++++++++------------ ci/deps/actions-311.yaml | 56 +++++++++++----------- ci/deps/actions-38-downstream_compat.yaml | 58 +++++++++++------------ ci/deps/actions-38.yaml | 58 +++++++++++------------ ci/deps/actions-39.yaml | 58 +++++++++++------------ ci/deps/circle-38-arm64.yaml | 56 +++++++++++----------- environment.yml | 56 +++++++++++----------- requirements-dev.txt | 56 +++++++++++----------- 8 files changed, 228 insertions(+), 228 deletions(-) diff --git a/ci/deps/actions-310.yaml b/ci/deps/actions-310.yaml index b500cf66b10c2..57929eb7a1ab1 100644 --- a/ci/deps/actions-310.yaml +++ b/ci/deps/actions-310.yaml @@ -22,36 +22,36 @@ dependencies: - pytz # optional dependencies - - beautifulsoup4 + - beautifulsoup4>=4.9.3 - blosc - - bottleneck - - brotlipy - - fastparquet - - fsspec - - html5lib - - hypothesis - - gcsfs - - jinja2 - - lxml + - bottleneck>=1.3.2 + - brotlipy>=0.7.0 + - fastparquet>=0.6.3 + - fsspec>=2021.07.0 + - html5lib>=1.1 + - hypothesis>=6.34.2 + - gcsfs>=2021.07.0 + - jinja2>=3.0.0 + - lxml>=4.6.3 - matplotlib>=3.6.1 - - numba - - numexpr - - openpyxl - - odfpy - - pandas-gbq - - psycopg2 - - pymysql - - pytables - - pyarrow - - pyreadstat - - python-snappy - - pyxlsb + - numba>=0.53.1 + - numexpr>=2.7.3 + - openpyxl>=3.0.7 + - odfpy>=1.4.1 + - pandas-gbq>=0.15.0 + - psycopg2>=2.8.6 + - pymysql>=1.0.2 + - pytables>=3.6.1 + - pyarrow>=6.0.0 + - pyreadstat>=1.1.2 + - python-snappy>=0.6.0 + - pyxlsb>=1.0.8 - s3fs>=2021.08.0 - - scipy - - sqlalchemy<1.4.46 - - tabulate + - scipy>=1.7.1 + - sqlalchemy<1.4.46, >=1.4.16 + - tabulate>=0.8.9 - tzdata>=2022a - - xarray - - xlrd - - xlsxwriter - - zstandard + - xarray>=0.21.0 + - xlrd>=2.0.1 + - xlsxwriter>=1.4.3 + - zstandard>=0.15.2 diff --git a/ci/deps/actions-311.yaml b/ci/deps/actions-311.yaml index 8e15c7b4740c5..b0adfeab04504 100644 --- a/ci/deps/actions-311.yaml +++ b/ci/deps/actions-311.yaml @@ -9,7 +9,7 @@ dependencies: - cython>=0.29.32 # test dependencies - - pytest>=7.0 + - pytest>=7.0.0 - pytest-cov - pytest-xdist>=2.2.0 - psutil @@ -22,36 +22,36 @@ dependencies: - pytz # optional dependencies - - beautifulsoup4 + - beautifulsoup4>=4.9.3 - blosc - - bottleneck - - brotlipy - - fastparquet - - fsspec - - html5lib - - hypothesis - - gcsfs - - jinja2 - - lxml + - bottleneck>=1.3.2 + - brotlipy>=0.7.0 + - fastparquet>=0.6.3 + - fsspec>=2021.07.0 + - html5lib>=1.1 + - hypothesis>=6.34.2 + - gcsfs>=2021.07.0 + - jinja2>=3.0.0 + - lxml>=4.6.3 - matplotlib>=3.6.1 # - numba not compatible with 3.11 - - numexpr - - openpyxl - - odfpy - - pandas-gbq - - psycopg2 - - pymysql + - numexpr>=2.7.3 + - openpyxl>=3.0.7 + - odfpy>=1.4.1 + - pandas-gbq>=0.15.0 + - psycopg2>=2.8.6 + - pymysql>=1.0.2 # - pytables>=3.8.0 # first version that supports 3.11 - - pyarrow - - pyreadstat - - python-snappy - - pyxlsb + - pyarrow>=6.0.0 + - pyreadstat>=1.1.2 + - python-snappy>=0.6.0 + - pyxlsb>=1.0.8 - s3fs>=2021.08.0 - - scipy - - sqlalchemy<1.4.46 - - tabulate + - scipy>=1.7.1 + - sqlalchemy<1.4.46, >=1.4.16 + - tabulate>=0.8.9 - tzdata>=2022a - - xarray - - xlrd - - xlsxwriter - - zstandard + - xarray>=0.21.0 + - xlrd>=2.0.1 + - xlsxwriter>=1.4.3 + - zstandard>=0.15.2 diff --git a/ci/deps/actions-38-downstream_compat.yaml b/ci/deps/actions-38-downstream_compat.yaml index 151cabfacb434..5d5cc5f69714f 100644 --- a/ci/deps/actions-38-downstream_compat.yaml +++ b/ci/deps/actions-38-downstream_compat.yaml @@ -23,37 +23,37 @@ dependencies: - pytz # optional dependencies - - beautifulsoup4 + - beautifulsoup4>=4.9.3 - blosc - - brotlipy - - bottleneck - - fastparquet - - fsspec - - html5lib - - hypothesis - - gcsfs - - jinja2 - - lxml + - brotlipy>=0.7.0 + - bottleneck>=1.3.2 + - fastparquet>=0.6.3 + - fsspec>=2021.07.0 + - html5lib>=1.1 + - hypothesis>=6.34.2 + - gcsfs>=2021.07.0 + - jinja2>=3.0.0 + - lxml>=4.6.3 - matplotlib>=3.6.1 - - numba - - numexpr - - openpyxl - - odfpy - - psycopg2 - - pyarrow - - pymysql - - pyreadstat - - pytables - - python-snappy - - pyxlsb + - numba>=0.53.1 + - numexpr>=2.7.3 + - openpyxl>=3.0.7 + - odfpy>=1.4.1 + - psycopg2>=2.8.6 + - pyarrow>=6.0.0 + - pymysql>=1.0.2 + - pyreadstat>=1.1.2 + - pytables>=3.6.1 + - python-snappy>=0.6.0 + - pyxlsb>=1.0.8 - s3fs>=2021.08.0 - - scipy - - sqlalchemy<1.4.46 - - tabulate - - xarray - - xlrd - - xlsxwriter - - zstandard + - scipy>=1.7.1 + - sqlalchemy<1.4.46, >=1.4.16 + - tabulate>=0.8.9 + - xarray>=0.21.0 + - xlrd>=2.0.1 + - xlsxwriter>=1.4.3 + - zstandard>=0.15.2 # downstream packages - botocore @@ -66,6 +66,6 @@ dependencies: - statsmodels - coverage - pandas-datareader - - pandas-gbq + - pandas-gbq>=0.15.0 - pyyaml - py diff --git a/ci/deps/actions-38.yaml b/ci/deps/actions-38.yaml index bd1246ddc7a3e..a0453df30288d 100644 --- a/ci/deps/actions-38.yaml +++ b/ci/deps/actions-38.yaml @@ -22,35 +22,35 @@ dependencies: - pytz # optional dependencies - - beautifulsoup4 + - beautifulsoup4>=4.9.3 - blosc - - bottleneck - - brotlipy - - fastparquet - - fsspec - - html5lib - - hypothesis - - gcsfs - - jinja2 - - lxml + - bottleneck>=1.3.2 + - brotlipy>=0.7.0 + - fastparquet>=0.6.3 + - fsspec>=2021.07.0 + - html5lib>=1.1 + - hypothesis>=6.34.2 + - gcsfs>=2021.07.0 + - jinja2>=3.0.0 + - lxml>=4.6.3 - matplotlib>=3.6.1 - - numba - - numexpr - - openpyxl - - odfpy - - pandas-gbq - - psycopg2 - - pyarrow - - pymysql - - pyreadstat - - pytables - - python-snappy - - pyxlsb + - numba>=0.53.1 + - numexpr>=2.7.3 + - openpyxl>=3.0.7 + - odfpy>=1.4.1 + - pandas-gbq>=0.15.0 + - psycopg2>=2.8.6 + - pyarrow>=6.0.0 + - pymysql>=1.0.2 + - pyreadstat>=1.1.2 + - pytables>=3.6.1 + - python-snappy>=0.6.0 + - pyxlsb>=1.0.8 - s3fs>=2021.08.0 - - scipy - - sqlalchemy<1.4.46 - - tabulate - - xarray - - xlrd - - xlsxwriter - - zstandard + - scipy>=1.7.1 + - sqlalchemy<1.4.46, >=1.4.16 + - tabulate>=0.8.9 + - xarray>=0.21.0 + - xlrd>=2.0.1 + - xlsxwriter>=1.4.3 + - zstandard>=0.15.2 diff --git a/ci/deps/actions-39.yaml b/ci/deps/actions-39.yaml index 93b73b20591b0..d8841c470a1ae 100644 --- a/ci/deps/actions-39.yaml +++ b/ci/deps/actions-39.yaml @@ -22,36 +22,36 @@ dependencies: - pytz # optional dependencies - - beautifulsoup4 + - beautifulsoup4>=4.9.3 - blosc - - bottleneck - - brotlipy - - fastparquet - - fsspec - - html5lib - - hypothesis - - gcsfs - - jinja2 - - lxml + - bottleneck>=1.3.2 + - brotlipy>=0.7.0 + - fastparquet>=0.6.3 + - fsspec>=2021.07.0 + - html5lib>=1.1 + - hypothesis>=6.34.2 + - gcsfs>=2021.07.0 + - jinja2>=3.0.0 + - lxml>=4.6.3 - matplotlib>=3.6.1 - - numba - - numexpr - - openpyxl - - odfpy - - pandas-gbq - - psycopg2 - - pymysql - - pyarrow - - pyreadstat - - pytables - - python-snappy - - pyxlsb + - numba>=0.53.1 + - numexpr>=2.7.3 + - openpyxl>=3.0.7 + - odfpy>=1.4.1 + - pandas-gbq>=0.15.0 + - psycopg2>=2.8.6 + - pymysql>=1.0.2 + - pyarrow>=6.0.0 + - pyreadstat>=1.1.2 + - pytables>=3.6.1 + - python-snappy>=0.6.0 + - pyxlsb>=1.0.8 - s3fs>=2021.08.0 - - scipy - - sqlalchemy<1.4.46 - - tabulate + - scipy>=1.7.1 + - sqlalchemy<1.4.46, >=1.4.16 + - tabulate>=0.8.9 - tzdata>=2022a - - xarray - - xlrd - - xlsxwriter - - zstandard + - xarray>=0.21.0 + - xlrd>=2.0.1 + - xlsxwriter>=1.4.3 + - zstandard>=0.15.2 diff --git a/ci/deps/circle-38-arm64.yaml b/ci/deps/circle-38-arm64.yaml index addbda194cc0c..32d63cab341e1 100644 --- a/ci/deps/circle-38-arm64.yaml +++ b/ci/deps/circle-38-arm64.yaml @@ -22,36 +22,36 @@ dependencies: - pytz # optional dependencies - - beautifulsoup4 + - beautifulsoup4>=4.9.3 - blosc - - bottleneck - - brotlipy - - fastparquet - - fsspec - - html5lib - - hypothesis - - gcsfs - - jinja2 - - lxml + - bottleneck>=1.3.2 + - brotlipy>=0.7.0 + - fastparquet>=0.6.3 + - fsspec>=2021.07.0 + - html5lib>=1.1 + - hypothesis>=6.34.2 + - gcsfs>=2021.07.0 + - jinja2>=3.0.0 + - lxml>=4.6.3 - matplotlib>=3.6.1 - - numba - - numexpr - - openpyxl - - odfpy - - pandas-gbq - - psycopg2 - - pyarrow - - pymysql + - numba>=0.53.1 + - numexpr>=2.7.3 + - openpyxl>=3.0.7 + - odfpy>=1.4.1 + - pandas-gbq>=0.15.0 + - psycopg2>=2.8.6 + - pyarrow>=6.0.0 + - pymysql>=1.0.2 # Not provided on ARM #- pyreadstat - - pytables - - python-snappy - - pyxlsb + - pytables>=3.6.1 + - python-snappy>=0.6.0 + - pyxlsb>=1.0.8 - s3fs>=2021.08.0 - - scipy - - sqlalchemy<1.4.46 - - tabulate - - xarray - - xlrd - - xlsxwriter - - zstandard + - scipy>=1.7.1 + - sqlalchemy<1.4.46, >=1.4.16 + - tabulate>=0.8.9 + - xarray>=0.21.0 + - xlrd>=2.0.1 + - xlsxwriter>=1.4.3 + - zstandard>=0.15.2 diff --git a/environment.yml b/environment.yml index 9169cbf08b45d..c768d31b02831 100644 --- a/environment.yml +++ b/environment.yml @@ -24,40 +24,40 @@ dependencies: - pytz # optional dependencies - - beautifulsoup4 + - beautifulsoup4>=4.9.3 - blosc - - brotlipy - - bottleneck - - fastparquet - - fsspec - - html5lib - - hypothesis - - gcsfs + - brotlipy>=0.7.0 + - bottleneck>=1.3.2 + - fastparquet>=0.6.3 + - fsspec>=2021.07.0 + - html5lib>=1.1 + - hypothesis>=6.34.2 + - gcsfs>=2021.07.0 - ipython - - jinja2 - - lxml + - jinja2>=3.0.0 + - lxml>=4.6.3 - matplotlib>=3.6.1 - numba>=0.53.1 - - numexpr>=2.8.0 # pin for "Run checks on imported code" job - - openpyxl - - odfpy + - numexpr>=2.7.3 # pin for "Run checks on imported code" job + - openpyxl>=3.0.7 + - odfpy>=1.4.1 - py - - psycopg2 - - pyarrow<11 - - pymysql - - pyreadstat - - pytables - - python-snappy - - pyxlsb + - psycopg2>=2.8.6 + - pyarrow<11, >=6.0.0 + - pymysql>=1.0.2 + - pyreadstat>=1.1.2 + - pytables>=3.6.1 + - python-snappy>=0.6.0 + - pyxlsb>=1.0.8 - s3fs>=2021.08.0 - - scipy - - sqlalchemy - - tabulate + - scipy>=1.7.1 + - sqlalchemy>=1.4.16 + - tabulate>=0.8.9 - tzdata>=2022a - - xarray - - xlrd - - xlsxwriter - - zstandard + - xarray>=0.21.0 + - xlrd>=2.0.1 + - xlsxwriter>=1.4.3 + - zstandard>=0.15.2 # downstream packages - dask-core @@ -109,7 +109,7 @@ dependencies: - ipykernel # web - - jinja2 # in optional dependencies, but documented here as needed + - jinja2>=3.0.0 # in optional dependencies, but documented here as needed - markdown - feedparser - pyyaml diff --git a/requirements-dev.txt b/requirements-dev.txt index b6992a7266600..0fcd14cecc191 100644 --- a/requirements-dev.txt +++ b/requirements-dev.txt @@ -13,40 +13,40 @@ coverage python-dateutil numpy pytz -beautifulsoup4 +beautifulsoup4>=4.9.3 blosc -brotlipy -bottleneck -fastparquet -fsspec -html5lib -hypothesis -gcsfs +brotlipy>=0.7.0 +bottleneck>=1.3.2 +fastparquet>=0.6.3 +fsspec>=2021.07.0 +html5lib>=1.1 +hypothesis>=6.34.2 +gcsfs>=2021.07.0 ipython -jinja2 -lxml +jinja2>=3.0.0 +lxml>=4.6.3 matplotlib>=3.6.1 numba>=0.53.1 -numexpr>=2.8.0 -openpyxl -odfpy +numexpr>=2.7.3 +openpyxl>=3.0.7 +odfpy>=1.4.1 py -psycopg2-binary -pyarrow<11 -pymysql -pyreadstat -tables -python-snappy -pyxlsb +psycopg2-binary>=2.8.6 +pyarrow<11, >=6.0.0 +pymysql>=1.0.2 +pyreadstat>=1.1.2 +tables>=3.6.1 +python-snappy>=0.6.0 +pyxlsb>=1.0.8 s3fs>=2021.08.0 -scipy -sqlalchemy -tabulate +scipy>=1.7.1 +sqlalchemy>=1.4.16 +tabulate>=0.8.9 tzdata>=2022.1 -xarray -xlrd -xlsxwriter -zstandard +xarray>=0.21.0 +xlrd>=2.0.1 +xlsxwriter>=1.4.3 +zstandard>=0.15.2 dask seaborn moto @@ -80,7 +80,7 @@ ipywidgets nbformat notebook>=6.0.3 ipykernel -jinja2 +jinja2>=3.0.0 markdown feedparser pyyaml From 3a7357330d1dba204bf2bfaef68d19ac150d2c1d Mon Sep 17 00:00:00 2001 From: doge Date: Thu, 9 Feb 2023 15:17:33 -0800 Subject: [PATCH 12/33] Fix merge issue --- ci/deps/actions-310.yaml | 2 +- ci/deps/actions-311.yaml | 2 +- ci/deps/actions-38-downstream_compat.yaml | 4 ++-- ci/deps/actions-38.yaml | 16 ++-------------- ci/deps/actions-39.yaml | 2 +- ci/deps/circle-38-arm64.yaml | 2 +- 6 files changed, 8 insertions(+), 20 deletions(-) diff --git a/ci/deps/actions-310.yaml b/ci/deps/actions-310.yaml index 57929eb7a1ab1..b81c346936ce0 100644 --- a/ci/deps/actions-310.yaml +++ b/ci/deps/actions-310.yaml @@ -42,7 +42,7 @@ dependencies: - psycopg2>=2.8.6 - pymysql>=1.0.2 - pytables>=3.6.1 - - pyarrow>=6.0.0 + - pyarrow>=7.0.0 - pyreadstat>=1.1.2 - python-snappy>=0.6.0 - pyxlsb>=1.0.8 diff --git a/ci/deps/actions-311.yaml b/ci/deps/actions-311.yaml index b0adfeab04504..29e52278e942e 100644 --- a/ci/deps/actions-311.yaml +++ b/ci/deps/actions-311.yaml @@ -42,7 +42,7 @@ dependencies: - psycopg2>=2.8.6 - pymysql>=1.0.2 # - pytables>=3.8.0 # first version that supports 3.11 - - pyarrow>=6.0.0 + - pyarrow>=7.0.0 - pyreadstat>=1.1.2 - python-snappy>=0.6.0 - pyxlsb>=1.0.8 diff --git a/ci/deps/actions-38-downstream_compat.yaml b/ci/deps/actions-38-downstream_compat.yaml index 5d5cc5f69714f..06bffd3236837 100644 --- a/ci/deps/actions-38-downstream_compat.yaml +++ b/ci/deps/actions-38-downstream_compat.yaml @@ -40,7 +40,7 @@ dependencies: - openpyxl>=3.0.7 - odfpy>=1.4.1 - psycopg2>=2.8.6 - - pyarrow>=6.0.0 + - pyarrow<11, >=7.0.0 - pymysql>=1.0.2 - pyreadstat>=1.1.2 - pytables>=3.6.1 @@ -48,7 +48,7 @@ dependencies: - pyxlsb>=1.0.8 - s3fs>=2021.08.0 - scipy>=1.7.1 - - sqlalchemy<1.4.46, >=1.4.16 + - sqlalchemy>=1.4.16 - tabulate>=0.8.9 - xarray>=0.21.0 - xlrd>=2.0.1 diff --git a/ci/deps/actions-38.yaml b/ci/deps/actions-38.yaml index 79cf18c53727e..2c50c4e367ebd 100644 --- a/ci/deps/actions-38.yaml +++ b/ci/deps/actions-38.yaml @@ -34,25 +34,13 @@ dependencies: - jinja2>=3.0.0 - lxml>=4.6.3 - matplotlib>=3.6.1 - - numba - - numexpr - - openpyxl - - odfpy - - pandas-gbq - - psycopg2 - - pyarrow<11 - - pymysql - - pyreadstat - - pytables - - python-snappy - - pyxlsb - numba>=0.53.1 - numexpr>=2.7.3 - openpyxl>=3.0.7 - odfpy>=1.4.1 - pandas-gbq>=0.15.0 - psycopg2>=2.8.6 - - pyarrow>=6.0.0 + - pyarrow<11, >=7.0.0 - pymysql>=1.0.2 - pyreadstat>=1.1.2 - pytables>=3.6.1 @@ -60,7 +48,7 @@ dependencies: - pyxlsb>=1.0.8 - s3fs>=2021.08.0 - scipy>=1.7.1 - - sqlalchemy<1.4.46, >=1.4.16 + - sqlalchemy>=1.4.16 - tabulate>=0.8.9 - xarray>=0.21.0 - xlrd>=2.0.1 diff --git a/ci/deps/actions-39.yaml b/ci/deps/actions-39.yaml index d8841c470a1ae..3d1d2e55ee7dc 100644 --- a/ci/deps/actions-39.yaml +++ b/ci/deps/actions-39.yaml @@ -41,7 +41,7 @@ dependencies: - pandas-gbq>=0.15.0 - psycopg2>=2.8.6 - pymysql>=1.0.2 - - pyarrow>=6.0.0 + - pyarrow>=7.0.0 - pyreadstat>=1.1.2 - pytables>=3.6.1 - python-snappy>=0.6.0 diff --git a/ci/deps/circle-38-arm64.yaml b/ci/deps/circle-38-arm64.yaml index 32d63cab341e1..5dd3f7160eec7 100644 --- a/ci/deps/circle-38-arm64.yaml +++ b/ci/deps/circle-38-arm64.yaml @@ -40,7 +40,7 @@ dependencies: - odfpy>=1.4.1 - pandas-gbq>=0.15.0 - psycopg2>=2.8.6 - - pyarrow>=6.0.0 + - pyarrow>=7.0.0 - pymysql>=1.0.2 # Not provided on ARM #- pyreadstat From 0968c10c3fe3fbbef9a14167fe6d0c866c4a80ba Mon Sep 17 00:00:00 2001 From: doge Date: Tue, 14 Feb 2023 18:37:37 -0800 Subject: [PATCH 13/33] add parameter for custom toml dependency --- scripts/validate_min_versions_in_sync.py | 35 +++++++++++++----------- 1 file changed, 19 insertions(+), 16 deletions(-) diff --git a/scripts/validate_min_versions_in_sync.py b/scripts/validate_min_versions_in_sync.py index 9835cf6cf513b..9cc4293cfc499 100755 --- a/scripts/validate_min_versions_in_sync.py +++ b/scripts/validate_min_versions_in_sync.py @@ -49,19 +49,25 @@ def pin_min_versions_to_ci_deps(): all_yaml_files = list(YAML_PATH.iterdir()) all_yaml_files.append(ENV_PATH) + with open(SETUP_PATH, "rb") as toml_f: + toml_dic = tomllib.load(toml_f) for curr_file in all_yaml_files: with open(curr_file) as yaml_f: - data = update_yaml_file_version(yaml_f) + data = update_yaml_file_version(yaml_f, toml_dic) os.remove(curr_file) with open(curr_file, "w") as f: f.write(data) -def update_yaml_file_version(yaml_file): +def update_yaml_file_version(yaml_file, toml_dic): exclusion_list = { "python=3.8[build=*_pypy]": None, } - toml_dependencies = get_versions_from_toml() + toml_deps = {} + toml_dependencies = set(toml_dic["project"]["optional-dependencies"]["all"]) + for dependency in toml_dependencies: + toml_package, toml_version = dependency.strip().split(">=") + toml_deps[toml_package] = toml_version data = yaml_file.read() yaml_f = yaml.safe_load(data) yaml_deps = yaml_f["dependencies"] @@ -90,37 +96,35 @@ def update_yaml_file_version(yaml_file): yaml_left = yaml_package else: yaml_package, yaml_version = str(dep_line).strip().split(operator) - if operator == "<" or operator == ">": - if yaml_package in toml_dependencies: + if "<" in operator or ">" in operator: + if yaml_package in toml_deps: if version.parse(yaml_version) <= version.parse( - toml_dependencies[yaml_package] + toml_deps[yaml_package] ): yaml_left = yaml_package else: yaml_left = str(dep_line) + ", " else: yaml_left = yaml_package + operator - if yaml_package in toml_dependencies: + if yaml_package in toml_deps: if ">" in yaml_left or "<" in yaml_left: if "," in yaml_left: # ex: "numpy<1.24.0," + ">=" + "1.2" - replace_text = yaml_left + ">=" + toml_dependencies[yaml_package] + replace_text = yaml_left + ">=" + toml_deps[yaml_package] else: - replace_text = yaml_left + toml_dependencies[yaml_package] + replace_text = yaml_left + toml_deps[yaml_package] # update yaml package version to TOML min version elif yaml_version is not None: - if version.parse(toml_dependencies[yaml_package]) > version.parse( - yaml_version - ): + if version.parse(toml_deps[yaml_package]) > version.parse(yaml_version): # ex: "hypothesis>=" + "6.34.2" - replace_text = yaml_left + toml_dependencies[yaml_package] - elif version.parse(toml_dependencies[yaml_package]) == version.parse( + replace_text = yaml_left + toml_deps[yaml_package] + elif version.parse(toml_deps[yaml_package]) == version.parse( yaml_version ): replace_text = dep_line else: # ex: "hypothesis + ">=" + 6.34.2" - replace_text = yaml_package + ">=" + toml_dependencies[yaml_package] + replace_text = yaml_package + ">=" + toml_deps[yaml_package] data = data.replace(search_text, replace_text) return data @@ -176,7 +180,6 @@ def get_versions_from_toml() -> dict[str, str]: """Min versions in pyproject.toml for pip install pandas[extra].""" install_map = _optional.INSTALL_MAPPING optional_dependencies = {} - with open(SETUP_PATH, "rb") as pyproject_f: pyproject_toml = tomllib.load(pyproject_f) opt_deps = pyproject_toml["project"]["optional-dependencies"] From 6d30e63da824cb14595fbff308e4293dc762038d Mon Sep 17 00:00:00 2001 From: doge Date: Tue, 14 Feb 2023 18:42:16 -0800 Subject: [PATCH 14/33] Add dummy test files for test_validate_min_versions_in_sync.py --- scripts/tests/dummy_toml_file.toml | 537 ++++++++++++++++++ scripts/tests/dummy_yaml_expected_file_1.yaml | 56 ++ .../tests/dummy_yaml_unmodified_file_1.yaml | 56 ++ 3 files changed, 649 insertions(+) create mode 100644 scripts/tests/dummy_toml_file.toml create mode 100644 scripts/tests/dummy_yaml_expected_file_1.yaml create mode 100644 scripts/tests/dummy_yaml_unmodified_file_1.yaml diff --git a/scripts/tests/dummy_toml_file.toml b/scripts/tests/dummy_toml_file.toml new file mode 100644 index 0000000000000..97a5ce1180bfb --- /dev/null +++ b/scripts/tests/dummy_toml_file.toml @@ -0,0 +1,537 @@ +[build-system] +# Minimum requirements for the build system to execute. +# See https://github.com/scipy/scipy/pull/12940 for the AIX issue. +requires = [ + "setuptools>=61.0.0", + "wheel", + "Cython>=0.29.32,<3", # Note: sync with setup.py, environment.yml and asv.conf.json + "oldest-supported-numpy>=2022.8.16", + "versioneer[toml]" +] +# build-backend = "setuptools.build_meta" + +[project] +name = 'pandas' +dynamic = [ + 'version' +] +description = 'Powerful data structures for data analysis, time series, and statistics' +readme = 'README.md' +authors = [ + { name = 'The Pandas Development Team', email='pandas-dev@python.org' }, +] +license = {file = 'LICENSE'} +requires-python = '>=3.8' +dependencies = [ + "numpy>=1.20.3; python_version<'3.10'", + "numpy>=1.21.0; python_version>='3.10'", + "numpy>=1.23.2; python_version>='3.11'", + "python-dateutil>=2.8.2", + "pytz>=2020.1" +] +classifiers = [ + 'Development Status :: 5 - Production/Stable', + 'Environment :: Console', + 'Intended Audience :: Science/Research', + 'License :: OSI Approved :: BSD License', + 'Operating System :: OS Independent', + 'Programming Language :: Cython', + 'Programming Language :: Python', + 'Programming Language :: Python :: 3', + 'Programming Language :: Python :: 3 :: Only', + 'Programming Language :: Python :: 3.8', + 'Programming Language :: Python :: 3.9', + 'Programming Language :: Python :: 3.10', + 'Programming Language :: Python :: 3.11', + 'Topic :: Scientific/Engineering' +] + +[project.urls] +homepage = 'https://pandas.pydata.org' +documentation = 'https://pandas.pydata.org/docs/' +repository = 'https://github.com/pandas-dev/pandas' + +[project.entry-points."pandas_plotting_backends"] +matplotlib = "pandas:plotting._matplotlib" + +[project.optional-dependencies] +test = ['hypothesis>=6.34.2', 'pytest>=7.0.0', 'pytest-xdist>=2.2.0', 'pytest-asyncio>=0.17.0'] +performance = ['bottleneck>=1.3.2', 'numba>=0.53.1', 'numexpr>=2.7.1'] +timezone = ['tzdata>=2022.1'] +computation = ['scipy>=1.7.1', 'xarray>=0.21.0'] +fss = ['fsspec>=2021.07.0'] +aws = ['s3fs>=2021.08.0'] +gcp = ['gcsfs>=2021.07.0', 'pandas-gbq>=0.15.0'] +excel = ['odfpy>=1.4.1', 'openpyxl>=3.0.7', 'pyxlsb>=1.0.8', 'xlrd>=2.0.1', 'xlsxwriter>=1.4.3'] +parquet = ['pyarrow>=7.0.0'] +feather = ['pyarrow>=7.0.0'] +hdf5 = [# blosc only available on conda (https://github.com/Blosc/python-blosc/issues/297) + #'blosc>=1.20.1', + 'tables>=3.6.1'] +spss = ['pyreadstat>=1.1.2'] +postgresql = ['SQLAlchemy>=1.4.16', 'psycopg2>=2.8.6'] +mysql = ['SQLAlchemy>=1.4.16', 'pymysql>=1.0.2'] +sql-other = ['SQLAlchemy>=1.4.16'] +html = ['beautifulsoup4>=4.9.3', 'html5lib>=1.1', 'lxml>=4.6.3'] +xml = ['lxml>=4.6.3'] +plot = ['matplotlib>=3.6.1'] +output_formatting = ['jinja2>=3.0.0', 'tabulate>=0.8.9'] +clipboard = ['PyQt5>=5.15.1', 'qtpy>=2.2.0'] +compression = ['brotlipy>=0.7.0', 'python-snappy>=0.6.0', 'zstandard>=0.15.2'] +all = ['beautifulsoup4>=5.9.3', + # blosc only available on conda (https://github.com/Blosc/python-blosc/issues/297) + #'blosc>=1.21.0', + 'bottleneck>=1.3.2', + 'brotlipy>=0.7.0', + 'fastparquet>=0.6.3', + 'fsspec>=2021.07.0', + 'gcsfs>=2021.07.0', + 'html5lib>=1.1', + 'hypothesis>=6.34.2', + 'jinja2>=3.0.0', + 'lxml>=4.6.3', + 'matplotlib>=3.6.1', + 'numba>=0.53.1', + 'numexpr>=2.7.3', + 'odfpy>=1.4.1', + 'openpyxl>=3.0.7', + 'pandas-gbq>=0.15.0', + 'psycopg2>=2.8.6', + 'pyarrow>=7.0.0', + 'pymysql>=1.0.2', + 'PyQt5>=5.15.1', + 'pyreadstat>=1.1.2', + 'pytest>=7.0.0', + 'pytest-xdist>=2.2.0', + 'pytest-asyncio>=0.17.0', + 'python-snappy>=0.6.0', + 'pyxlsb>=1.0.8', + 'qtpy>=2.2.0', + 'scipy>=1.7.1', + 's3fs>=2021.08.0', + 'SQLAlchemy>=1.4.16', + 'tables>=3.6.1', + 'tabulate>=0.8.9', + 'tzdata>=2022.1', + 'xarray>=0.21.0', + 'xlrd>=2.0.1', + 'xlsxwriter>=1.4.3', + 'zstandard>=0.15.2'] + +# TODO: Remove after setuptools support is dropped. +[tool.setuptools] +include-package-data = true + +[tool.setuptools.packages.find] +include = ["pandas", "pandas.*"] +namespaces = false + +[tool.setuptools.exclude-package-data] +"*" = ["*.c", "*.h"] + +# See the docstring in versioneer.py for instructions. Note that you must +# re-run 'versioneer.py setup' after changing this section, and commit the +# resulting files. +[tool.versioneer] +VCS = "git" +style = "pep440" +versionfile_source = "pandas/_version.py" +versionfile_build = "pandas/_version.py" +tag_prefix = "v" +parentdir_prefix = "pandas-" + +[tool.cibuildwheel] +skip = "cp36-* cp37-* pp37-* *-manylinux_i686 *_ppc64le *_s390x *-musllinux*" +build-verbosity = "3" +test-requires = "hypothesis>=6.34.2 pytest>=7.0.0 pytest-xdist>=2.2.0 pytest-asyncio>=0.17" +test-command = "python {project}/ci/test_wheels.py" + +[tool.cibuildwheel.macos] +archs = "x86_64 arm64" +test-skip = "*_arm64" + +[tool.cibuildwheel.windows] +repair-wheel-command = "python ci/fix_wheels.py {wheel} {dest_dir}" + +[[tool.cibuildwheel.overrides]] +select = "*-win*" +# We test separately for Windows, since we use +# the base windows docker image to check if any dlls are +# missing from the wheel +test-command = "" + +[[tool.cibuildwheel.overrides]] +select = "*-win32" +environment = { IS_32_BIT="true" } + +[tool.black] +target-version = ['py38', 'py39'] +exclude = ''' +( + asv_bench/env + | \.egg + | \.git + | \.hg + | \.mypy_cache + | \.nox + | \.tox + | \.venv + | _build + | buck-out + | build + | dist + | setup.py +) +''' + +[tool.ruff] +line-length = 88 +update-check = false +target-version = "py38" + +select = [ + # pyflakes + "F", + # pycodestyle + "E", + "W", + # flake8-2020 + "YTT", + # flake8-bugbear + "B", + # flake8-quotes + "Q", + # pylint + "PLE", "PLR", "PLW", +] + +ignore = [ + # space before : (needed for how black formats slicing) + # "E203", # not yet implemented + # module level import not at top of file + "E402", + # do not assign a lambda expression, use a def + "E731", + # line break before binary operator + # "W503", # not yet implemented + # line break after binary operator + # "W504", # not yet implemented + # controversial + "B006", + # controversial + "B007", + # controversial + "B008", + # setattr is used to side-step mypy + "B009", + # getattr is used to side-step mypy + "B010", + # tests use assert False + "B011", + # tests use comparisons but not their returned value + "B015", + # false positives + "B019", + # Loop control variable overrides iterable it iterates + "B020", + # Function definition does not bind loop variable + "B023", + # Functions defined inside a loop must not use variables redefined in the loop + # "B301", # not yet implemented + + # Additional checks that don't pass yet + # Within an except clause, raise exceptions with ... + "B904", +] + +exclude = [ + "doc/sphinxext/*.py", + "doc/build/*.py", + "doc/temp/*.py", + ".eggs/*.py", + "versioneer.py", + # exclude asv benchmark environments from linting + "env", +] + +[tool.pylint.messages_control] +max-line-length = 88 +disable = [ + # intentionally turned off + "broad-except", + "c-extension-no-member", + "comparison-with-itself", + "import-error", + "import-outside-toplevel", + "invalid-name", + "invalid-unary-operand-type", + "line-too-long", + "no-else-continue", + "no-else-raise", + "no-else-return", + "no-member", + "no-name-in-module", + "not-an-iterable", + "overridden-final-method", + "pointless-statement", + "redundant-keyword-arg", + "singleton-comparison", + "too-many-ancestors", + "too-many-arguments", + "too-many-boolean-expressions", + "too-many-branches", + "too-many-function-args", + "too-many-instance-attributes", + "too-many-locals", + "too-many-nested-blocks", + "too-many-public-methods", + "too-many-return-statements", + "too-many-statements", + "unexpected-keyword-arg", + "ungrouped-imports", + "unsubscriptable-object", + "unsupported-assignment-operation", + "unsupported-membership-test", + "unused-import", + "use-implicit-booleaness-not-comparison", + "use-implicit-booleaness-not-len", + "wrong-import-order", + "wrong-import-position", + + # misc + "abstract-class-instantiated", + "no-value-for-parameter", + "undefined-variable", + "unpacking-non-sequence", + + # pylint type "C": convention, for programming standard violation + "missing-class-docstring", + "missing-function-docstring", + "missing-module-docstring", + "too-many-lines", + "unidiomatic-typecheck", + "unnecessary-dunder-call", + "unnecessary-lambda-assignment", + + # pylint type "R": refactor, for bad code smell + "consider-using-with", + "cyclic-import", + "duplicate-code", + "inconsistent-return-statements", + "redefined-argument-from-local", + "too-few-public-methods", + + # pylint type "W": warning, for python specific problems + "abstract-method", + "arguments-differ", + "arguments-out-of-order", + "arguments-renamed", + "attribute-defined-outside-init", + "comparison-with-callable", + "dangerous-default-value", + "deprecated-module", + "eval-used", + "expression-not-assigned", + "fixme", + "global-statement", + "invalid-overridden-method", + "keyword-arg-before-vararg", + "possibly-unused-variable", + "protected-access", + "raise-missing-from", + "redefined-builtin", + "redefined-outer-name", + "self-cls-assignment", + "signature-differs", + "super-init-not-called", + "try-except-raise", + "unnecessary-lambda", + "unspecified-encoding", + "unused-argument", + "unused-variable", + "using-constant-test" +] + +[tool.pytest.ini_options] +# sync minversion with pyproject.toml & install.rst +minversion = "7.0" +addopts = "--strict-data-files --strict-markers --strict-config --capture=no --durations=30 --junitxml=test-data.xml" +empty_parameter_set_mark = "fail_at_collect" +xfail_strict = true +testpaths = "pandas" +doctest_optionflags = [ + "NORMALIZE_WHITESPACE", + "IGNORE_EXCEPTION_DETAIL", + "ELLIPSIS", +] +filterwarnings = [ + # Will be fixed in numba 0.56: https://github.com/numba/numba/issues/7758 + "ignore:`np.MachAr` is deprecated:DeprecationWarning:numba", + "ignore:.*urllib3:DeprecationWarning:botocore", + "ignore:Setuptools is replacing distutils.:UserWarning:_distutils_hack", + # https://github.com/PyTables/PyTables/issues/822 + "ignore:a closed node found in the registry:UserWarning:tables", + "ignore:`np.object` is a deprecated:DeprecationWarning:tables", + "ignore:tostring:DeprecationWarning:tables", + "ignore:distutils Version classes are deprecated:DeprecationWarning:numexpr", + "ignore:distutils Version classes are deprecated:DeprecationWarning:fastparquet", + "ignore:distutils Version classes are deprecated:DeprecationWarning:fsspec", +] +junit_family = "xunit2" +markers = [ + "single_cpu: tests that should run on a single cpu only", + "slow: mark a test as slow", + "network: mark a test as network", + "db: tests requiring a database (mysql or postgres)", + "clipboard: mark a pd.read_clipboard test", + "arm_slow: mark a test as slow for arm64 architecture", + "arraymanager: mark a test to run with ArrayManager enabled", +] +asyncio_mode = "strict" + +[tool.mypy] +# Import discovery +mypy_path = "typings" +files = ["pandas", "typings"] +namespace_packages = false +explicit_package_bases = false +ignore_missing_imports = true +follow_imports = "normal" +follow_imports_for_stubs = false +no_site_packages = false +no_silence_site_packages = false +# Platform configuration +python_version = "3.8" +platform = "linux-64" +# Disallow dynamic typing +disallow_any_unimported = false # TODO +disallow_any_expr = false # TODO +disallow_any_decorated = false # TODO +disallow_any_explicit = false # TODO +disallow_any_generics = false # TODO +disallow_subclassing_any = false # TODO +# Untyped definitions and calls +disallow_untyped_calls = false # TODO +disallow_untyped_defs = false # TODO +disallow_incomplete_defs = false # TODO +check_untyped_defs = true +disallow_untyped_decorators = true +# None and Optional handling +no_implicit_optional = true +strict_optional = true +# Configuring warnings +warn_redundant_casts = true +warn_unused_ignores = true +warn_no_return = true +warn_return_any = false # TODO +warn_unreachable = false # GH#27396 +# Suppressing errors +ignore_errors = false +enable_error_code = "ignore-without-code" +# Miscellaneous strictness flags +allow_untyped_globals = false +allow_redefinition = false +local_partial_types = false +implicit_reexport = true +strict_equality = true +# Configuring error messages +show_error_context = false +show_column_numbers = false +show_error_codes = true + +[[tool.mypy.overrides]] +module = [ + "pandas.tests.*", + "pandas._version", + "pandas.io.clipboard", +] +check_untyped_defs = false + +[[tool.mypy.overrides]] +module = [ + "pandas.tests.apply.test_series_apply", + "pandas.tests.arithmetic.conftest", + "pandas.tests.arrays.sparse.test_combine_concat", + "pandas.tests.dtypes.test_common", + "pandas.tests.frame.methods.test_to_records", + "pandas.tests.groupby.test_rank", + "pandas.tests.groupby.transform.test_transform", + "pandas.tests.indexes.interval.test_interval", + "pandas.tests.indexing.test_categorical", + "pandas.tests.io.excel.test_writers", + "pandas.tests.reductions.test_reductions", + "pandas.tests.test_expressions", +] +ignore_errors = true + +# To be kept consistent with "Import Formatting" section in contributing.rst +[tool.isort] +known_pre_libs = "pandas._config" +known_pre_core = ["pandas._libs", "pandas._typing", "pandas.util._*", "pandas.compat", "pandas.errors"] +known_dtypes = "pandas.core.dtypes" +known_post_core = ["pandas.tseries", "pandas.io", "pandas.plotting"] +sections = ["FUTURE", "STDLIB", "THIRDPARTY" ,"PRE_LIBS" , "PRE_CORE", "DTYPES", "FIRSTPARTY", "POST_CORE", "LOCALFOLDER"] +profile = "black" +combine_as_imports = true +force_grid_wrap = 2 +force_sort_within_sections = true +skip_glob = "env" +skip = "pandas/__init__.py" + +[tool.pyright] +pythonVersion = "3.8" +typeCheckingMode = "basic" +include = ["pandas", "typings"] +exclude = ["pandas/tests", "pandas/io/clipboard", "pandas/util/version"] +# enable subset of "strict" +reportDuplicateImport = true +reportInvalidStubStatement = true +reportOverlappingOverload = true +reportPropertyTypeMismatch = true +reportUntypedClassDecorator = true +reportUntypedFunctionDecorator = true +reportUntypedNamedTuple = true +reportUnusedImport = true +# disable subset of "basic" +reportGeneralTypeIssues = false +reportMissingModuleSource = false +reportOptionalCall = false +reportOptionalIterable = false +reportOptionalMemberAccess = false +reportOptionalOperand = false +reportOptionalSubscript = false +reportPrivateImportUsage = false +reportUnboundVariable = false + +[tool.coverage.run] +branch = true +omit = ["pandas/_typing.py", "pandas/_version.py"] +plugins = ["Cython.Coverage"] +source = ["pandas"] + +[tool.coverage.report] +ignore_errors = false +show_missing = true +omit = ["pandas/_version.py"] +exclude_lines = [ + # Have to re-enable the standard pragma + "pragma: no cover", + # Don't complain about missing debug-only code:s + "def __repr__", + "if self.debug", + # Don't complain if tests don't hit defensive assertion code: + "raise AssertionError", + "raise NotImplementedError", + "AbstractMethodError", + # Don't complain if non-runnable code isn't run: + "if 0:", + "if __name__ == .__main__.:", + "if TYPE_CHECKING:", +] + +[tool.coverage.html] +directory = "coverage_html_report" + +[tool.codespell] +ignore-words-list = "blocs, coo, hist, nd, sav, ser, recuse" +ignore-regex = 'https://([\w/\.])+' diff --git a/scripts/tests/dummy_yaml_expected_file_1.yaml b/scripts/tests/dummy_yaml_expected_file_1.yaml new file mode 100644 index 0000000000000..e55edee5a5b82 --- /dev/null +++ b/scripts/tests/dummy_yaml_expected_file_1.yaml @@ -0,0 +1,56 @@ +name: pandas-dev +channels: + - conda-forge +dependencies: + - python=3.8 + + # build dependencies + - versioneer[toml] + - cython>=0.29.32 + + # test dependencies + - pytest>=7.0.0 + - pytest-cov + - pytest-xdist>=2.2.0 + - psutil + - pytest-asyncio>=0.17 + - boto3 + + # required dependencies + - python-dateutil + - numpy + - pytz + + # optional dependencies + - beautifulsoup4>=5.9.3 + - blosc + - bottleneck>=1.3.2 + - brotlipy>=0.7.0 + - fastparquet>=0.6.3 + - fsspec>=2021.07.0 + - html5lib>=1.1 + - hypothesis>=6.34.2 + - gcsfs>=2021.07.0 + - jinja2>=3.0.0 + - lxml>=4.6.3 + - matplotlib>=3.6.1 + - numba>=0.53.1 + - numexpr>=2.7.3 + - openpyxl>=3.0.7 + - odfpy>=1.4.1 + - pandas-gbq>=0.15.0 + - psycopg2>=2.8.6 + - pyarrow<11, >=7.0.0 + - pymysql>=1.0.2 + - pyreadstat>=1.1.2 + - pytables>=3.6.1 + - python-snappy>=0.6.0 + - pyxlsb>=1.0.8 + - s3fs>=2021.08.0 + - scipy>=1.7.1 + - sqlalchemy>=1.4.16 + - tabulate>=0.8.9 + - xarray>=0.21.0 + - xlrd>=2.0.1 + - xlsxwriter>=1.4.3 + - zstandard>=0.15.2 diff --git a/scripts/tests/dummy_yaml_unmodified_file_1.yaml b/scripts/tests/dummy_yaml_unmodified_file_1.yaml new file mode 100644 index 0000000000000..45c4b332c74e8 --- /dev/null +++ b/scripts/tests/dummy_yaml_unmodified_file_1.yaml @@ -0,0 +1,56 @@ +name: pandas-dev +channels: + - conda-forge +dependencies: + - python=3.8 + + # build dependencies + - versioneer[toml] + - cython>=0.29.32 + + # test dependencies + - pytest>=7.0.0 + - pytest-cov + - pytest-xdist>=2.2.0 + - psutil + - pytest-asyncio>=0.17 + - boto3 + + # required dependencies + - python-dateutil + - numpy + - pytz + + # optional dependencies + - beautifulsoup4 + - blosc + - bottleneck>=1.3.2 + - brotlipy + - fastparquet>=0.6.3 + - fsspec>=2021.07.0 + - html5lib>=1.1 + - hypothesis + - gcsfs>=2021.07.0 + - jinja2 + - lxml>=4.6.3 + - matplotlib>=3.6.1 + - numba + - numexpr>=2.7.3 + - openpyxl>=3.0.7 + - odfpy>=1.4.1 + - pandas-gbq>=0.15.0 + - psycopg2 + - pyarrow<11, >=7.0.0 + - pymysql>=1.0.2 + - pyreadstat>=1.1.2 + - pytables>=3.6.1 + - python-snappy>=0.6.0 + - pyxlsb>=1.0.8 + - s3fs>=2021.08.0 + - scipy>=1.7.1 + - sqlalchemy>=1.4.16 + - tabulate>=0.8.9 + - xarray>=0.21.0 + - xlrd>=2.0.1 + - xlsxwriter>=1.4.3 + - zstandard>=0.15.2 From f923b987359c2e458e13ce8489ccccd38779d41c Mon Sep 17 00:00:00 2001 From: doge Date: Tue, 14 Feb 2023 18:49:03 -0800 Subject: [PATCH 15/33] Add unit test: test_update_yaml_file_version() --- .../test_validate_min_versions_in_sync.py | 29 +++++++++++++++++++ 1 file changed, 29 insertions(+) create mode 100644 scripts/tests/test_validate_min_versions_in_sync.py diff --git a/scripts/tests/test_validate_min_versions_in_sync.py b/scripts/tests/test_validate_min_versions_in_sync.py new file mode 100644 index 0000000000000..c5bba80a622b5 --- /dev/null +++ b/scripts/tests/test_validate_min_versions_in_sync.py @@ -0,0 +1,29 @@ +import pathlib +import sys + +if sys.version_info >= (3, 11): + import tomllib +else: + import tomli as tomllib + +from scripts.validate_min_versions_in_sync import update_yaml_file_version + + +def test_update_yaml_file_version(): + # minimum dependencies file + TOML_PATH = pathlib.Path("scripts/tests/dummy_toml_file.toml") + # yaml file that needs to be corrected + YAML_UNMODIFIED_PATH = pathlib.Path( + "scripts/tests/dummy_yaml_unmodified_file_1.yaml" + ) + # yaml file that is the expected result + YAML_EXPECTED_PATH = pathlib.Path("scripts/tests/dummy_yaml_expected_file_1.yaml") + toml_dic = {} + with open(TOML_PATH, "rb") as toml_f: + toml_dic = tomllib.load(toml_f) + with open(YAML_UNMODIFIED_PATH) as yaml_f: + result_yaml_file = update_yaml_file_version(yaml_f, toml_dic) + with open(YAML_EXPECTED_PATH) as yaml_f: + dummy_yaml_expected_file_1 = yaml_f.read() + # if assert passes, then yaml file version update is correct + assert result_yaml_file == dummy_yaml_expected_file_1 From 1431ae82eb85577f1d20a0968b9dfa44baa0f84f Mon Sep 17 00:00:00 2001 From: doge Date: Wed, 15 Feb 2023 11:44:11 -0800 Subject: [PATCH 16/33] Add dummy test files for edge cases --- .../dummy_test_files/dummy_toml_file.toml | 537 ++++++++++++++++++ .../dummy_yaml_expected_file_1.yaml | 57 ++ .../dummy_yaml_expected_file_2.yaml | 3 + .../dummy_yaml_expected_file_3.yaml | 4 + .../dummy_yaml_expected_file_4.yaml | 5 + .../dummy_yaml_expected_file_5.yaml | 6 + .../dummy_yaml_unmodified_file_1.yaml | 57 ++ .../dummy_yaml_unmodified_file_2.yaml | 3 + .../dummy_yaml_unmodified_file_3.yaml | 4 + .../dummy_yaml_unmodified_file_4.yaml | 5 + .../dummy_yaml_unmodified_file_5.yaml | 6 + 11 files changed, 687 insertions(+) create mode 100644 scripts/tests/dummy_test_files/dummy_toml_file.toml create mode 100644 scripts/tests/dummy_test_files/dummy_yaml_expected_file_1.yaml create mode 100644 scripts/tests/dummy_test_files/dummy_yaml_expected_file_2.yaml create mode 100644 scripts/tests/dummy_test_files/dummy_yaml_expected_file_3.yaml create mode 100644 scripts/tests/dummy_test_files/dummy_yaml_expected_file_4.yaml create mode 100644 scripts/tests/dummy_test_files/dummy_yaml_expected_file_5.yaml create mode 100644 scripts/tests/dummy_test_files/dummy_yaml_unmodified_file_1.yaml create mode 100644 scripts/tests/dummy_test_files/dummy_yaml_unmodified_file_2.yaml create mode 100644 scripts/tests/dummy_test_files/dummy_yaml_unmodified_file_3.yaml create mode 100644 scripts/tests/dummy_test_files/dummy_yaml_unmodified_file_4.yaml create mode 100644 scripts/tests/dummy_test_files/dummy_yaml_unmodified_file_5.yaml diff --git a/scripts/tests/dummy_test_files/dummy_toml_file.toml b/scripts/tests/dummy_test_files/dummy_toml_file.toml new file mode 100644 index 0000000000000..97a5ce1180bfb --- /dev/null +++ b/scripts/tests/dummy_test_files/dummy_toml_file.toml @@ -0,0 +1,537 @@ +[build-system] +# Minimum requirements for the build system to execute. +# See https://github.com/scipy/scipy/pull/12940 for the AIX issue. +requires = [ + "setuptools>=61.0.0", + "wheel", + "Cython>=0.29.32,<3", # Note: sync with setup.py, environment.yml and asv.conf.json + "oldest-supported-numpy>=2022.8.16", + "versioneer[toml]" +] +# build-backend = "setuptools.build_meta" + +[project] +name = 'pandas' +dynamic = [ + 'version' +] +description = 'Powerful data structures for data analysis, time series, and statistics' +readme = 'README.md' +authors = [ + { name = 'The Pandas Development Team', email='pandas-dev@python.org' }, +] +license = {file = 'LICENSE'} +requires-python = '>=3.8' +dependencies = [ + "numpy>=1.20.3; python_version<'3.10'", + "numpy>=1.21.0; python_version>='3.10'", + "numpy>=1.23.2; python_version>='3.11'", + "python-dateutil>=2.8.2", + "pytz>=2020.1" +] +classifiers = [ + 'Development Status :: 5 - Production/Stable', + 'Environment :: Console', + 'Intended Audience :: Science/Research', + 'License :: OSI Approved :: BSD License', + 'Operating System :: OS Independent', + 'Programming Language :: Cython', + 'Programming Language :: Python', + 'Programming Language :: Python :: 3', + 'Programming Language :: Python :: 3 :: Only', + 'Programming Language :: Python :: 3.8', + 'Programming Language :: Python :: 3.9', + 'Programming Language :: Python :: 3.10', + 'Programming Language :: Python :: 3.11', + 'Topic :: Scientific/Engineering' +] + +[project.urls] +homepage = 'https://pandas.pydata.org' +documentation = 'https://pandas.pydata.org/docs/' +repository = 'https://github.com/pandas-dev/pandas' + +[project.entry-points."pandas_plotting_backends"] +matplotlib = "pandas:plotting._matplotlib" + +[project.optional-dependencies] +test = ['hypothesis>=6.34.2', 'pytest>=7.0.0', 'pytest-xdist>=2.2.0', 'pytest-asyncio>=0.17.0'] +performance = ['bottleneck>=1.3.2', 'numba>=0.53.1', 'numexpr>=2.7.1'] +timezone = ['tzdata>=2022.1'] +computation = ['scipy>=1.7.1', 'xarray>=0.21.0'] +fss = ['fsspec>=2021.07.0'] +aws = ['s3fs>=2021.08.0'] +gcp = ['gcsfs>=2021.07.0', 'pandas-gbq>=0.15.0'] +excel = ['odfpy>=1.4.1', 'openpyxl>=3.0.7', 'pyxlsb>=1.0.8', 'xlrd>=2.0.1', 'xlsxwriter>=1.4.3'] +parquet = ['pyarrow>=7.0.0'] +feather = ['pyarrow>=7.0.0'] +hdf5 = [# blosc only available on conda (https://github.com/Blosc/python-blosc/issues/297) + #'blosc>=1.20.1', + 'tables>=3.6.1'] +spss = ['pyreadstat>=1.1.2'] +postgresql = ['SQLAlchemy>=1.4.16', 'psycopg2>=2.8.6'] +mysql = ['SQLAlchemy>=1.4.16', 'pymysql>=1.0.2'] +sql-other = ['SQLAlchemy>=1.4.16'] +html = ['beautifulsoup4>=4.9.3', 'html5lib>=1.1', 'lxml>=4.6.3'] +xml = ['lxml>=4.6.3'] +plot = ['matplotlib>=3.6.1'] +output_formatting = ['jinja2>=3.0.0', 'tabulate>=0.8.9'] +clipboard = ['PyQt5>=5.15.1', 'qtpy>=2.2.0'] +compression = ['brotlipy>=0.7.0', 'python-snappy>=0.6.0', 'zstandard>=0.15.2'] +all = ['beautifulsoup4>=5.9.3', + # blosc only available on conda (https://github.com/Blosc/python-blosc/issues/297) + #'blosc>=1.21.0', + 'bottleneck>=1.3.2', + 'brotlipy>=0.7.0', + 'fastparquet>=0.6.3', + 'fsspec>=2021.07.0', + 'gcsfs>=2021.07.0', + 'html5lib>=1.1', + 'hypothesis>=6.34.2', + 'jinja2>=3.0.0', + 'lxml>=4.6.3', + 'matplotlib>=3.6.1', + 'numba>=0.53.1', + 'numexpr>=2.7.3', + 'odfpy>=1.4.1', + 'openpyxl>=3.0.7', + 'pandas-gbq>=0.15.0', + 'psycopg2>=2.8.6', + 'pyarrow>=7.0.0', + 'pymysql>=1.0.2', + 'PyQt5>=5.15.1', + 'pyreadstat>=1.1.2', + 'pytest>=7.0.0', + 'pytest-xdist>=2.2.0', + 'pytest-asyncio>=0.17.0', + 'python-snappy>=0.6.0', + 'pyxlsb>=1.0.8', + 'qtpy>=2.2.0', + 'scipy>=1.7.1', + 's3fs>=2021.08.0', + 'SQLAlchemy>=1.4.16', + 'tables>=3.6.1', + 'tabulate>=0.8.9', + 'tzdata>=2022.1', + 'xarray>=0.21.0', + 'xlrd>=2.0.1', + 'xlsxwriter>=1.4.3', + 'zstandard>=0.15.2'] + +# TODO: Remove after setuptools support is dropped. +[tool.setuptools] +include-package-data = true + +[tool.setuptools.packages.find] +include = ["pandas", "pandas.*"] +namespaces = false + +[tool.setuptools.exclude-package-data] +"*" = ["*.c", "*.h"] + +# See the docstring in versioneer.py for instructions. Note that you must +# re-run 'versioneer.py setup' after changing this section, and commit the +# resulting files. +[tool.versioneer] +VCS = "git" +style = "pep440" +versionfile_source = "pandas/_version.py" +versionfile_build = "pandas/_version.py" +tag_prefix = "v" +parentdir_prefix = "pandas-" + +[tool.cibuildwheel] +skip = "cp36-* cp37-* pp37-* *-manylinux_i686 *_ppc64le *_s390x *-musllinux*" +build-verbosity = "3" +test-requires = "hypothesis>=6.34.2 pytest>=7.0.0 pytest-xdist>=2.2.0 pytest-asyncio>=0.17" +test-command = "python {project}/ci/test_wheels.py" + +[tool.cibuildwheel.macos] +archs = "x86_64 arm64" +test-skip = "*_arm64" + +[tool.cibuildwheel.windows] +repair-wheel-command = "python ci/fix_wheels.py {wheel} {dest_dir}" + +[[tool.cibuildwheel.overrides]] +select = "*-win*" +# We test separately for Windows, since we use +# the base windows docker image to check if any dlls are +# missing from the wheel +test-command = "" + +[[tool.cibuildwheel.overrides]] +select = "*-win32" +environment = { IS_32_BIT="true" } + +[tool.black] +target-version = ['py38', 'py39'] +exclude = ''' +( + asv_bench/env + | \.egg + | \.git + | \.hg + | \.mypy_cache + | \.nox + | \.tox + | \.venv + | _build + | buck-out + | build + | dist + | setup.py +) +''' + +[tool.ruff] +line-length = 88 +update-check = false +target-version = "py38" + +select = [ + # pyflakes + "F", + # pycodestyle + "E", + "W", + # flake8-2020 + "YTT", + # flake8-bugbear + "B", + # flake8-quotes + "Q", + # pylint + "PLE", "PLR", "PLW", +] + +ignore = [ + # space before : (needed for how black formats slicing) + # "E203", # not yet implemented + # module level import not at top of file + "E402", + # do not assign a lambda expression, use a def + "E731", + # line break before binary operator + # "W503", # not yet implemented + # line break after binary operator + # "W504", # not yet implemented + # controversial + "B006", + # controversial + "B007", + # controversial + "B008", + # setattr is used to side-step mypy + "B009", + # getattr is used to side-step mypy + "B010", + # tests use assert False + "B011", + # tests use comparisons but not their returned value + "B015", + # false positives + "B019", + # Loop control variable overrides iterable it iterates + "B020", + # Function definition does not bind loop variable + "B023", + # Functions defined inside a loop must not use variables redefined in the loop + # "B301", # not yet implemented + + # Additional checks that don't pass yet + # Within an except clause, raise exceptions with ... + "B904", +] + +exclude = [ + "doc/sphinxext/*.py", + "doc/build/*.py", + "doc/temp/*.py", + ".eggs/*.py", + "versioneer.py", + # exclude asv benchmark environments from linting + "env", +] + +[tool.pylint.messages_control] +max-line-length = 88 +disable = [ + # intentionally turned off + "broad-except", + "c-extension-no-member", + "comparison-with-itself", + "import-error", + "import-outside-toplevel", + "invalid-name", + "invalid-unary-operand-type", + "line-too-long", + "no-else-continue", + "no-else-raise", + "no-else-return", + "no-member", + "no-name-in-module", + "not-an-iterable", + "overridden-final-method", + "pointless-statement", + "redundant-keyword-arg", + "singleton-comparison", + "too-many-ancestors", + "too-many-arguments", + "too-many-boolean-expressions", + "too-many-branches", + "too-many-function-args", + "too-many-instance-attributes", + "too-many-locals", + "too-many-nested-blocks", + "too-many-public-methods", + "too-many-return-statements", + "too-many-statements", + "unexpected-keyword-arg", + "ungrouped-imports", + "unsubscriptable-object", + "unsupported-assignment-operation", + "unsupported-membership-test", + "unused-import", + "use-implicit-booleaness-not-comparison", + "use-implicit-booleaness-not-len", + "wrong-import-order", + "wrong-import-position", + + # misc + "abstract-class-instantiated", + "no-value-for-parameter", + "undefined-variable", + "unpacking-non-sequence", + + # pylint type "C": convention, for programming standard violation + "missing-class-docstring", + "missing-function-docstring", + "missing-module-docstring", + "too-many-lines", + "unidiomatic-typecheck", + "unnecessary-dunder-call", + "unnecessary-lambda-assignment", + + # pylint type "R": refactor, for bad code smell + "consider-using-with", + "cyclic-import", + "duplicate-code", + "inconsistent-return-statements", + "redefined-argument-from-local", + "too-few-public-methods", + + # pylint type "W": warning, for python specific problems + "abstract-method", + "arguments-differ", + "arguments-out-of-order", + "arguments-renamed", + "attribute-defined-outside-init", + "comparison-with-callable", + "dangerous-default-value", + "deprecated-module", + "eval-used", + "expression-not-assigned", + "fixme", + "global-statement", + "invalid-overridden-method", + "keyword-arg-before-vararg", + "possibly-unused-variable", + "protected-access", + "raise-missing-from", + "redefined-builtin", + "redefined-outer-name", + "self-cls-assignment", + "signature-differs", + "super-init-not-called", + "try-except-raise", + "unnecessary-lambda", + "unspecified-encoding", + "unused-argument", + "unused-variable", + "using-constant-test" +] + +[tool.pytest.ini_options] +# sync minversion with pyproject.toml & install.rst +minversion = "7.0" +addopts = "--strict-data-files --strict-markers --strict-config --capture=no --durations=30 --junitxml=test-data.xml" +empty_parameter_set_mark = "fail_at_collect" +xfail_strict = true +testpaths = "pandas" +doctest_optionflags = [ + "NORMALIZE_WHITESPACE", + "IGNORE_EXCEPTION_DETAIL", + "ELLIPSIS", +] +filterwarnings = [ + # Will be fixed in numba 0.56: https://github.com/numba/numba/issues/7758 + "ignore:`np.MachAr` is deprecated:DeprecationWarning:numba", + "ignore:.*urllib3:DeprecationWarning:botocore", + "ignore:Setuptools is replacing distutils.:UserWarning:_distutils_hack", + # https://github.com/PyTables/PyTables/issues/822 + "ignore:a closed node found in the registry:UserWarning:tables", + "ignore:`np.object` is a deprecated:DeprecationWarning:tables", + "ignore:tostring:DeprecationWarning:tables", + "ignore:distutils Version classes are deprecated:DeprecationWarning:numexpr", + "ignore:distutils Version classes are deprecated:DeprecationWarning:fastparquet", + "ignore:distutils Version classes are deprecated:DeprecationWarning:fsspec", +] +junit_family = "xunit2" +markers = [ + "single_cpu: tests that should run on a single cpu only", + "slow: mark a test as slow", + "network: mark a test as network", + "db: tests requiring a database (mysql or postgres)", + "clipboard: mark a pd.read_clipboard test", + "arm_slow: mark a test as slow for arm64 architecture", + "arraymanager: mark a test to run with ArrayManager enabled", +] +asyncio_mode = "strict" + +[tool.mypy] +# Import discovery +mypy_path = "typings" +files = ["pandas", "typings"] +namespace_packages = false +explicit_package_bases = false +ignore_missing_imports = true +follow_imports = "normal" +follow_imports_for_stubs = false +no_site_packages = false +no_silence_site_packages = false +# Platform configuration +python_version = "3.8" +platform = "linux-64" +# Disallow dynamic typing +disallow_any_unimported = false # TODO +disallow_any_expr = false # TODO +disallow_any_decorated = false # TODO +disallow_any_explicit = false # TODO +disallow_any_generics = false # TODO +disallow_subclassing_any = false # TODO +# Untyped definitions and calls +disallow_untyped_calls = false # TODO +disallow_untyped_defs = false # TODO +disallow_incomplete_defs = false # TODO +check_untyped_defs = true +disallow_untyped_decorators = true +# None and Optional handling +no_implicit_optional = true +strict_optional = true +# Configuring warnings +warn_redundant_casts = true +warn_unused_ignores = true +warn_no_return = true +warn_return_any = false # TODO +warn_unreachable = false # GH#27396 +# Suppressing errors +ignore_errors = false +enable_error_code = "ignore-without-code" +# Miscellaneous strictness flags +allow_untyped_globals = false +allow_redefinition = false +local_partial_types = false +implicit_reexport = true +strict_equality = true +# Configuring error messages +show_error_context = false +show_column_numbers = false +show_error_codes = true + +[[tool.mypy.overrides]] +module = [ + "pandas.tests.*", + "pandas._version", + "pandas.io.clipboard", +] +check_untyped_defs = false + +[[tool.mypy.overrides]] +module = [ + "pandas.tests.apply.test_series_apply", + "pandas.tests.arithmetic.conftest", + "pandas.tests.arrays.sparse.test_combine_concat", + "pandas.tests.dtypes.test_common", + "pandas.tests.frame.methods.test_to_records", + "pandas.tests.groupby.test_rank", + "pandas.tests.groupby.transform.test_transform", + "pandas.tests.indexes.interval.test_interval", + "pandas.tests.indexing.test_categorical", + "pandas.tests.io.excel.test_writers", + "pandas.tests.reductions.test_reductions", + "pandas.tests.test_expressions", +] +ignore_errors = true + +# To be kept consistent with "Import Formatting" section in contributing.rst +[tool.isort] +known_pre_libs = "pandas._config" +known_pre_core = ["pandas._libs", "pandas._typing", "pandas.util._*", "pandas.compat", "pandas.errors"] +known_dtypes = "pandas.core.dtypes" +known_post_core = ["pandas.tseries", "pandas.io", "pandas.plotting"] +sections = ["FUTURE", "STDLIB", "THIRDPARTY" ,"PRE_LIBS" , "PRE_CORE", "DTYPES", "FIRSTPARTY", "POST_CORE", "LOCALFOLDER"] +profile = "black" +combine_as_imports = true +force_grid_wrap = 2 +force_sort_within_sections = true +skip_glob = "env" +skip = "pandas/__init__.py" + +[tool.pyright] +pythonVersion = "3.8" +typeCheckingMode = "basic" +include = ["pandas", "typings"] +exclude = ["pandas/tests", "pandas/io/clipboard", "pandas/util/version"] +# enable subset of "strict" +reportDuplicateImport = true +reportInvalidStubStatement = true +reportOverlappingOverload = true +reportPropertyTypeMismatch = true +reportUntypedClassDecorator = true +reportUntypedFunctionDecorator = true +reportUntypedNamedTuple = true +reportUnusedImport = true +# disable subset of "basic" +reportGeneralTypeIssues = false +reportMissingModuleSource = false +reportOptionalCall = false +reportOptionalIterable = false +reportOptionalMemberAccess = false +reportOptionalOperand = false +reportOptionalSubscript = false +reportPrivateImportUsage = false +reportUnboundVariable = false + +[tool.coverage.run] +branch = true +omit = ["pandas/_typing.py", "pandas/_version.py"] +plugins = ["Cython.Coverage"] +source = ["pandas"] + +[tool.coverage.report] +ignore_errors = false +show_missing = true +omit = ["pandas/_version.py"] +exclude_lines = [ + # Have to re-enable the standard pragma + "pragma: no cover", + # Don't complain about missing debug-only code:s + "def __repr__", + "if self.debug", + # Don't complain if tests don't hit defensive assertion code: + "raise AssertionError", + "raise NotImplementedError", + "AbstractMethodError", + # Don't complain if non-runnable code isn't run: + "if 0:", + "if __name__ == .__main__.:", + "if TYPE_CHECKING:", +] + +[tool.coverage.html] +directory = "coverage_html_report" + +[tool.codespell] +ignore-words-list = "blocs, coo, hist, nd, sav, ser, recuse" +ignore-regex = 'https://([\w/\.])+' diff --git a/scripts/tests/dummy_test_files/dummy_yaml_expected_file_1.yaml b/scripts/tests/dummy_test_files/dummy_yaml_expected_file_1.yaml new file mode 100644 index 0000000000000..35e5c5eed76fa --- /dev/null +++ b/scripts/tests/dummy_test_files/dummy_yaml_expected_file_1.yaml @@ -0,0 +1,57 @@ +# Test: random +name: pandas-dev +channels: + - conda-forge +dependencies: + - python=3.8 + + # build dependencies + - versioneer[toml] + - cython>=0.29.32 + + # test dependencies + - pytest>=7.0.0 + - pytest-cov + - pytest-xdist>=2.2.0 + - psutil + - pytest-asyncio>=0.17 + - boto3 + + # required dependencies + - python-dateutil + - numpy + - pytz + + # optional dependencies + - beautifulsoup4>=5.9.3 + - blosc + - bottleneck>=1.3.2 + - brotlipy>=0.7.0 + - fastparquet>=0.6.3 + - fsspec>=2021.07.0 + - html5lib>=1.1 + - hypothesis>=6.34.2 + - gcsfs>=2021.07.0 + - jinja2>=3.0.0 + - lxml>=4.6.3 + - matplotlib>=3.6.1 + - numba>=0.53.1 + - numexpr>=2.7.3 + - openpyxl>=3.0.7 + - odfpy>=1.4.1 + - pandas-gbq>=0.15.0 + - psycopg2>=2.8.6 + - pyarrow<11, >=7.0.0 + - pymysql>=1.0.2 + - pyreadstat>=1.1.2 + - pytables>=3.6.1 + - python-snappy>=0.6.0 + - pyxlsb>=1.0.8 + - s3fs>=2021.08.0 + - scipy>=1.7.1 + - sqlalchemy>=1.4.16 + - tabulate>=0.8.9 + - xarray>=0.21.0 + - xlrd>=2.0.1 + - xlsxwriter>=1.4.3 + - zstandard>=0.15.2 diff --git a/scripts/tests/dummy_test_files/dummy_yaml_expected_file_2.yaml b/scripts/tests/dummy_test_files/dummy_yaml_expected_file_2.yaml new file mode 100644 index 0000000000000..e07b221ecd44f --- /dev/null +++ b/scripts/tests/dummy_test_files/dummy_yaml_expected_file_2.yaml @@ -0,0 +1,3 @@ +# Test: same version +dependencies: + - jinja2>=3.0.0 diff --git a/scripts/tests/dummy_test_files/dummy_yaml_expected_file_3.yaml b/scripts/tests/dummy_test_files/dummy_yaml_expected_file_3.yaml new file mode 100644 index 0000000000000..72721c2842707 --- /dev/null +++ b/scripts/tests/dummy_test_files/dummy_yaml_expected_file_3.yaml @@ -0,0 +1,4 @@ +# Test: duplicate package +dependencies: + - jinja2>=3.0.0 + - jinja2>=3.0.0 diff --git a/scripts/tests/dummy_test_files/dummy_yaml_expected_file_4.yaml b/scripts/tests/dummy_test_files/dummy_yaml_expected_file_4.yaml new file mode 100644 index 0000000000000..843e48330a928 --- /dev/null +++ b/scripts/tests/dummy_test_files/dummy_yaml_expected_file_4.yaml @@ -0,0 +1,5 @@ +# Test: empty version +dependencies: + - jinja2>=3.0.0 + - scipy>=1.7.1 + - SQLAlchemy>=1.4.16 diff --git a/scripts/tests/dummy_test_files/dummy_yaml_expected_file_5.yaml b/scripts/tests/dummy_test_files/dummy_yaml_expected_file_5.yaml new file mode 100644 index 0000000000000..d8b437ef36c37 --- /dev/null +++ b/scripts/tests/dummy_test_files/dummy_yaml_expected_file_5.yaml @@ -0,0 +1,6 @@ +# Test: range +dependencies: + - jinja2<8, >=3.0.0 + - scipy<9, >=1.7.1 + - SQLAlchemy<2.0, >=1.4.16 + - pyarrow<11, >=7.0.0 diff --git a/scripts/tests/dummy_test_files/dummy_yaml_unmodified_file_1.yaml b/scripts/tests/dummy_test_files/dummy_yaml_unmodified_file_1.yaml new file mode 100644 index 0000000000000..4ca758af1c8ad --- /dev/null +++ b/scripts/tests/dummy_test_files/dummy_yaml_unmodified_file_1.yaml @@ -0,0 +1,57 @@ +# Test: random +name: pandas-dev +channels: + - conda-forge +dependencies: + - python=3.8 + + # build dependencies + - versioneer[toml] + - cython>=0.29.32 + + # test dependencies + - pytest>=7.0.0 + - pytest-cov + - pytest-xdist>=2.2.0 + - psutil + - pytest-asyncio>=0.17 + - boto3 + + # required dependencies + - python-dateutil + - numpy + - pytz + + # optional dependencies + - beautifulsoup4 + - blosc + - bottleneck>=1.3.2 + - brotlipy + - fastparquet>=0.6.3 + - fsspec>=2021.07.0 + - html5lib>=1.1 + - hypothesis + - gcsfs>=2021.07.0 + - jinja2 + - lxml>=4.6.3 + - matplotlib>=3.6.1 + - numba + - numexpr>=2.7.3 + - openpyxl>=3.0.7 + - odfpy>=1.4.1 + - pandas-gbq>=0.15.0 + - psycopg2 + - pyarrow<11, >=7.0.0 + - pymysql>=1.0.2 + - pyreadstat>=1.1.2 + - pytables>=3.6.1 + - python-snappy>=0.6.0 + - pyxlsb>=1.0.8 + - s3fs>=2021.08.0 + - scipy>=1.7.1 + - sqlalchemy>=1.4.16 + - tabulate>=0.8.9 + - xarray>=0.21.0 + - xlrd>=2.0.1 + - xlsxwriter>=1.4.3 + - zstandard>=0.15.2 diff --git a/scripts/tests/dummy_test_files/dummy_yaml_unmodified_file_2.yaml b/scripts/tests/dummy_test_files/dummy_yaml_unmodified_file_2.yaml new file mode 100644 index 0000000000000..e07b221ecd44f --- /dev/null +++ b/scripts/tests/dummy_test_files/dummy_yaml_unmodified_file_2.yaml @@ -0,0 +1,3 @@ +# Test: same version +dependencies: + - jinja2>=3.0.0 diff --git a/scripts/tests/dummy_test_files/dummy_yaml_unmodified_file_3.yaml b/scripts/tests/dummy_test_files/dummy_yaml_unmodified_file_3.yaml new file mode 100644 index 0000000000000..72721c2842707 --- /dev/null +++ b/scripts/tests/dummy_test_files/dummy_yaml_unmodified_file_3.yaml @@ -0,0 +1,4 @@ +# Test: duplicate package +dependencies: + - jinja2>=3.0.0 + - jinja2>=3.0.0 diff --git a/scripts/tests/dummy_test_files/dummy_yaml_unmodified_file_4.yaml b/scripts/tests/dummy_test_files/dummy_yaml_unmodified_file_4.yaml new file mode 100644 index 0000000000000..c57b49a003efd --- /dev/null +++ b/scripts/tests/dummy_test_files/dummy_yaml_unmodified_file_4.yaml @@ -0,0 +1,5 @@ +# Test: empty version +dependencies: + - jinja2 + - scipy + - SQLAlchemy diff --git a/scripts/tests/dummy_test_files/dummy_yaml_unmodified_file_5.yaml b/scripts/tests/dummy_test_files/dummy_yaml_unmodified_file_5.yaml new file mode 100644 index 0000000000000..97c013141d099 --- /dev/null +++ b/scripts/tests/dummy_test_files/dummy_yaml_unmodified_file_5.yaml @@ -0,0 +1,6 @@ +# Test: range +dependencies: + - jinja2<8 + - scipy<9 + - SQLAlchemy<2.0 + - pyarrow<11 From d2d5d73baf0c2a5045b48585f29e1b2845289d52 Mon Sep 17 00:00:00 2001 From: doge Date: Wed, 15 Feb 2023 11:48:45 -0800 Subject: [PATCH 17/33] Add parametrization to unit test --- .../test_validate_min_versions_in_sync.py | 70 +++++++++++++++---- 1 file changed, 56 insertions(+), 14 deletions(-) diff --git a/scripts/tests/test_validate_min_versions_in_sync.py b/scripts/tests/test_validate_min_versions_in_sync.py index c5bba80a622b5..9556708175308 100644 --- a/scripts/tests/test_validate_min_versions_in_sync.py +++ b/scripts/tests/test_validate_min_versions_in_sync.py @@ -1,6 +1,8 @@ import pathlib import sys +import pytest + if sys.version_info >= (3, 11): import tomllib else: @@ -9,21 +11,61 @@ from scripts.validate_min_versions_in_sync import update_yaml_file_version -def test_update_yaml_file_version(): - # minimum dependencies file - TOML_PATH = pathlib.Path("scripts/tests/dummy_toml_file.toml") - # yaml file that needs to be corrected - YAML_UNMODIFIED_PATH = pathlib.Path( - "scripts/tests/dummy_yaml_unmodified_file_1.yaml" - ) - # yaml file that is the expected result - YAML_EXPECTED_PATH = pathlib.Path("scripts/tests/dummy_yaml_expected_file_1.yaml") - toml_dic = {} - with open(TOML_PATH, "rb") as toml_f: +@pytest.mark.parametrize( + "src_toml, src_yaml, expected_yaml", + [ + ( # random + pathlib.Path("scripts/tests/dummy_test_files/dummy_toml_file.toml"), + pathlib.Path( + "scripts/tests/dummy_test_files/dummy_yaml_unmodified_file_1.yaml" + ), + pathlib.Path( + "scripts/tests/dummy_test_files/dummy_yaml_expected_file_1.yaml" + ), + ), + ( # same version + pathlib.Path("scripts/tests/dummy_test_files/dummy_toml_file.toml"), + pathlib.Path( + "scripts/tests/dummy_test_files/dummy_yaml_unmodified_file_2.yaml" + ), + pathlib.Path( + "scripts/tests/dummy_test_files/dummy_yaml_expected_file_2.yaml" + ), + ), + ( # duplicate package + pathlib.Path("scripts/tests/dummy_test_files/dummy_toml_file.toml"), + pathlib.Path( + "scripts/tests/dummy_test_files/dummy_yaml_unmodified_file_3.yaml" + ), + pathlib.Path( + "scripts/tests/dummy_test_files/dummy_yaml_expected_file_3.yaml" + ), + ), + ( # empty version + pathlib.Path("scripts/tests/dummy_test_files/dummy_toml_file.toml"), + pathlib.Path( + "scripts/tests/dummy_test_files/dummy_yaml_unmodified_file_4.yaml" + ), + pathlib.Path( + "scripts/tests/dummy_test_files/dummy_yaml_expected_file_4.yaml" + ), + ), + ( # range version + pathlib.Path("scripts/tests/dummy_test_files/dummy_toml_file.toml"), + pathlib.Path( + "scripts/tests/dummy_test_files/dummy_yaml_unmodified_file_5.yaml" + ), + pathlib.Path( + "scripts/tests/dummy_test_files/dummy_yaml_expected_file_5.yaml" + ), + ), + ], +) +def test_update_yaml_file_version(src_toml, src_yaml, expected_yaml): + with open(src_toml, "rb") as toml_f: toml_dic = tomllib.load(toml_f) - with open(YAML_UNMODIFIED_PATH) as yaml_f: + with open(src_yaml) as yaml_f: result_yaml_file = update_yaml_file_version(yaml_f, toml_dic) - with open(YAML_EXPECTED_PATH) as yaml_f: + with open(expected_yaml) as yaml_f: dummy_yaml_expected_file_1 = yaml_f.read() - # if assert passes, then yaml file version update is correct assert result_yaml_file == dummy_yaml_expected_file_1 From 1af6771bda3666020379745ca3b695005b0b8a4f Mon Sep 17 00:00:00 2001 From: doge Date: Wed, 15 Feb 2023 12:22:22 -0800 Subject: [PATCH 18/33] Run pre-commit --- ci/deps/actions-310.yaml | 2 +- ci/deps/actions-311.yaml | 2 +- ci/deps/actions-38-minimum_versions.yaml | 2 +- ci/deps/actions-39.yaml | 2 +- ci/deps/actions-pypy-38.yaml | 2 +- environment.yml | 2 +- scripts/tests/dummy_toml_file.toml | 537 ------------------ scripts/tests/dummy_yaml_expected_file_1.yaml | 56 -- .../tests/dummy_yaml_unmodified_file_1.yaml | 56 -- 9 files changed, 6 insertions(+), 655 deletions(-) delete mode 100644 scripts/tests/dummy_toml_file.toml delete mode 100644 scripts/tests/dummy_yaml_expected_file_1.yaml delete mode 100644 scripts/tests/dummy_yaml_unmodified_file_1.yaml diff --git a/ci/deps/actions-310.yaml b/ci/deps/actions-310.yaml index b81c346936ce0..7011d8f1a5916 100644 --- a/ci/deps/actions-310.yaml +++ b/ci/deps/actions-310.yaml @@ -50,7 +50,7 @@ dependencies: - scipy>=1.7.1 - sqlalchemy<1.4.46, >=1.4.16 - tabulate>=0.8.9 - - tzdata>=2022a + - tzdata>=2022.1 - xarray>=0.21.0 - xlrd>=2.0.1 - xlsxwriter>=1.4.3 diff --git a/ci/deps/actions-311.yaml b/ci/deps/actions-311.yaml index f4b9cd06e9ae0..f2999db724361 100644 --- a/ci/deps/actions-311.yaml +++ b/ci/deps/actions-311.yaml @@ -50,7 +50,7 @@ dependencies: - scipy>=1.7.1 - sqlalchemy<1.4.46, >=1.4.16 - tabulate>=0.8.9 - - tzdata>=2022a + - tzdata>=2022.1 - xarray>=0.21.0 - xlrd>=2.0.1 - xlsxwriter>=1.4.3 diff --git a/ci/deps/actions-38-minimum_versions.yaml b/ci/deps/actions-38-minimum_versions.yaml index 7652b6347ad4f..706549d6af0cb 100644 --- a/ci/deps/actions-38-minimum_versions.yaml +++ b/ci/deps/actions-38-minimum_versions.yaml @@ -53,7 +53,7 @@ dependencies: - scipy=1.7.1 - sqlalchemy=1.4.16 - tabulate=0.8.9 - - tzdata=2022a + - tzdata=2022.1 - xarray=0.21.0 - xlrd=2.0.1 - xlsxwriter=1.4.3 diff --git a/ci/deps/actions-39.yaml b/ci/deps/actions-39.yaml index 3d1d2e55ee7dc..544550c1838a2 100644 --- a/ci/deps/actions-39.yaml +++ b/ci/deps/actions-39.yaml @@ -50,7 +50,7 @@ dependencies: - scipy>=1.7.1 - sqlalchemy<1.4.46, >=1.4.16 - tabulate>=0.8.9 - - tzdata>=2022a + - tzdata>=2022.1 - xarray>=0.21.0 - xlrd>=2.0.1 - xlsxwriter>=1.4.3 diff --git a/ci/deps/actions-pypy-38.yaml b/ci/deps/actions-pypy-38.yaml index 3218ec13a9c40..1fde1e733be5a 100644 --- a/ci/deps/actions-pypy-38.yaml +++ b/ci/deps/actions-pypy-38.yaml @@ -14,7 +14,7 @@ dependencies: # test dependencies - pytest>=7.0.0 - pytest-cov - - pytest-asyncio + - pytest-asyncio>=0.17.0 - pytest-xdist>=2.2.0 - hypothesis>=6.34.2 diff --git a/environment.yml b/environment.yml index c768d31b02831..76f83b6b26852 100644 --- a/environment.yml +++ b/environment.yml @@ -53,7 +53,7 @@ dependencies: - scipy>=1.7.1 - sqlalchemy>=1.4.16 - tabulate>=0.8.9 - - tzdata>=2022a + - tzdata>=2022.1 - xarray>=0.21.0 - xlrd>=2.0.1 - xlsxwriter>=1.4.3 diff --git a/scripts/tests/dummy_toml_file.toml b/scripts/tests/dummy_toml_file.toml deleted file mode 100644 index 97a5ce1180bfb..0000000000000 --- a/scripts/tests/dummy_toml_file.toml +++ /dev/null @@ -1,537 +0,0 @@ -[build-system] -# Minimum requirements for the build system to execute. -# See https://github.com/scipy/scipy/pull/12940 for the AIX issue. -requires = [ - "setuptools>=61.0.0", - "wheel", - "Cython>=0.29.32,<3", # Note: sync with setup.py, environment.yml and asv.conf.json - "oldest-supported-numpy>=2022.8.16", - "versioneer[toml]" -] -# build-backend = "setuptools.build_meta" - -[project] -name = 'pandas' -dynamic = [ - 'version' -] -description = 'Powerful data structures for data analysis, time series, and statistics' -readme = 'README.md' -authors = [ - { name = 'The Pandas Development Team', email='pandas-dev@python.org' }, -] -license = {file = 'LICENSE'} -requires-python = '>=3.8' -dependencies = [ - "numpy>=1.20.3; python_version<'3.10'", - "numpy>=1.21.0; python_version>='3.10'", - "numpy>=1.23.2; python_version>='3.11'", - "python-dateutil>=2.8.2", - "pytz>=2020.1" -] -classifiers = [ - 'Development Status :: 5 - Production/Stable', - 'Environment :: Console', - 'Intended Audience :: Science/Research', - 'License :: OSI Approved :: BSD License', - 'Operating System :: OS Independent', - 'Programming Language :: Cython', - 'Programming Language :: Python', - 'Programming Language :: Python :: 3', - 'Programming Language :: Python :: 3 :: Only', - 'Programming Language :: Python :: 3.8', - 'Programming Language :: Python :: 3.9', - 'Programming Language :: Python :: 3.10', - 'Programming Language :: Python :: 3.11', - 'Topic :: Scientific/Engineering' -] - -[project.urls] -homepage = 'https://pandas.pydata.org' -documentation = 'https://pandas.pydata.org/docs/' -repository = 'https://github.com/pandas-dev/pandas' - -[project.entry-points."pandas_plotting_backends"] -matplotlib = "pandas:plotting._matplotlib" - -[project.optional-dependencies] -test = ['hypothesis>=6.34.2', 'pytest>=7.0.0', 'pytest-xdist>=2.2.0', 'pytest-asyncio>=0.17.0'] -performance = ['bottleneck>=1.3.2', 'numba>=0.53.1', 'numexpr>=2.7.1'] -timezone = ['tzdata>=2022.1'] -computation = ['scipy>=1.7.1', 'xarray>=0.21.0'] -fss = ['fsspec>=2021.07.0'] -aws = ['s3fs>=2021.08.0'] -gcp = ['gcsfs>=2021.07.0', 'pandas-gbq>=0.15.0'] -excel = ['odfpy>=1.4.1', 'openpyxl>=3.0.7', 'pyxlsb>=1.0.8', 'xlrd>=2.0.1', 'xlsxwriter>=1.4.3'] -parquet = ['pyarrow>=7.0.0'] -feather = ['pyarrow>=7.0.0'] -hdf5 = [# blosc only available on conda (https://github.com/Blosc/python-blosc/issues/297) - #'blosc>=1.20.1', - 'tables>=3.6.1'] -spss = ['pyreadstat>=1.1.2'] -postgresql = ['SQLAlchemy>=1.4.16', 'psycopg2>=2.8.6'] -mysql = ['SQLAlchemy>=1.4.16', 'pymysql>=1.0.2'] -sql-other = ['SQLAlchemy>=1.4.16'] -html = ['beautifulsoup4>=4.9.3', 'html5lib>=1.1', 'lxml>=4.6.3'] -xml = ['lxml>=4.6.3'] -plot = ['matplotlib>=3.6.1'] -output_formatting = ['jinja2>=3.0.0', 'tabulate>=0.8.9'] -clipboard = ['PyQt5>=5.15.1', 'qtpy>=2.2.0'] -compression = ['brotlipy>=0.7.0', 'python-snappy>=0.6.0', 'zstandard>=0.15.2'] -all = ['beautifulsoup4>=5.9.3', - # blosc only available on conda (https://github.com/Blosc/python-blosc/issues/297) - #'blosc>=1.21.0', - 'bottleneck>=1.3.2', - 'brotlipy>=0.7.0', - 'fastparquet>=0.6.3', - 'fsspec>=2021.07.0', - 'gcsfs>=2021.07.0', - 'html5lib>=1.1', - 'hypothesis>=6.34.2', - 'jinja2>=3.0.0', - 'lxml>=4.6.3', - 'matplotlib>=3.6.1', - 'numba>=0.53.1', - 'numexpr>=2.7.3', - 'odfpy>=1.4.1', - 'openpyxl>=3.0.7', - 'pandas-gbq>=0.15.0', - 'psycopg2>=2.8.6', - 'pyarrow>=7.0.0', - 'pymysql>=1.0.2', - 'PyQt5>=5.15.1', - 'pyreadstat>=1.1.2', - 'pytest>=7.0.0', - 'pytest-xdist>=2.2.0', - 'pytest-asyncio>=0.17.0', - 'python-snappy>=0.6.0', - 'pyxlsb>=1.0.8', - 'qtpy>=2.2.0', - 'scipy>=1.7.1', - 's3fs>=2021.08.0', - 'SQLAlchemy>=1.4.16', - 'tables>=3.6.1', - 'tabulate>=0.8.9', - 'tzdata>=2022.1', - 'xarray>=0.21.0', - 'xlrd>=2.0.1', - 'xlsxwriter>=1.4.3', - 'zstandard>=0.15.2'] - -# TODO: Remove after setuptools support is dropped. -[tool.setuptools] -include-package-data = true - -[tool.setuptools.packages.find] -include = ["pandas", "pandas.*"] -namespaces = false - -[tool.setuptools.exclude-package-data] -"*" = ["*.c", "*.h"] - -# See the docstring in versioneer.py for instructions. Note that you must -# re-run 'versioneer.py setup' after changing this section, and commit the -# resulting files. -[tool.versioneer] -VCS = "git" -style = "pep440" -versionfile_source = "pandas/_version.py" -versionfile_build = "pandas/_version.py" -tag_prefix = "v" -parentdir_prefix = "pandas-" - -[tool.cibuildwheel] -skip = "cp36-* cp37-* pp37-* *-manylinux_i686 *_ppc64le *_s390x *-musllinux*" -build-verbosity = "3" -test-requires = "hypothesis>=6.34.2 pytest>=7.0.0 pytest-xdist>=2.2.0 pytest-asyncio>=0.17" -test-command = "python {project}/ci/test_wheels.py" - -[tool.cibuildwheel.macos] -archs = "x86_64 arm64" -test-skip = "*_arm64" - -[tool.cibuildwheel.windows] -repair-wheel-command = "python ci/fix_wheels.py {wheel} {dest_dir}" - -[[tool.cibuildwheel.overrides]] -select = "*-win*" -# We test separately for Windows, since we use -# the base windows docker image to check if any dlls are -# missing from the wheel -test-command = "" - -[[tool.cibuildwheel.overrides]] -select = "*-win32" -environment = { IS_32_BIT="true" } - -[tool.black] -target-version = ['py38', 'py39'] -exclude = ''' -( - asv_bench/env - | \.egg - | \.git - | \.hg - | \.mypy_cache - | \.nox - | \.tox - | \.venv - | _build - | buck-out - | build - | dist - | setup.py -) -''' - -[tool.ruff] -line-length = 88 -update-check = false -target-version = "py38" - -select = [ - # pyflakes - "F", - # pycodestyle - "E", - "W", - # flake8-2020 - "YTT", - # flake8-bugbear - "B", - # flake8-quotes - "Q", - # pylint - "PLE", "PLR", "PLW", -] - -ignore = [ - # space before : (needed for how black formats slicing) - # "E203", # not yet implemented - # module level import not at top of file - "E402", - # do not assign a lambda expression, use a def - "E731", - # line break before binary operator - # "W503", # not yet implemented - # line break after binary operator - # "W504", # not yet implemented - # controversial - "B006", - # controversial - "B007", - # controversial - "B008", - # setattr is used to side-step mypy - "B009", - # getattr is used to side-step mypy - "B010", - # tests use assert False - "B011", - # tests use comparisons but not their returned value - "B015", - # false positives - "B019", - # Loop control variable overrides iterable it iterates - "B020", - # Function definition does not bind loop variable - "B023", - # Functions defined inside a loop must not use variables redefined in the loop - # "B301", # not yet implemented - - # Additional checks that don't pass yet - # Within an except clause, raise exceptions with ... - "B904", -] - -exclude = [ - "doc/sphinxext/*.py", - "doc/build/*.py", - "doc/temp/*.py", - ".eggs/*.py", - "versioneer.py", - # exclude asv benchmark environments from linting - "env", -] - -[tool.pylint.messages_control] -max-line-length = 88 -disable = [ - # intentionally turned off - "broad-except", - "c-extension-no-member", - "comparison-with-itself", - "import-error", - "import-outside-toplevel", - "invalid-name", - "invalid-unary-operand-type", - "line-too-long", - "no-else-continue", - "no-else-raise", - "no-else-return", - "no-member", - "no-name-in-module", - "not-an-iterable", - "overridden-final-method", - "pointless-statement", - "redundant-keyword-arg", - "singleton-comparison", - "too-many-ancestors", - "too-many-arguments", - "too-many-boolean-expressions", - "too-many-branches", - "too-many-function-args", - "too-many-instance-attributes", - "too-many-locals", - "too-many-nested-blocks", - "too-many-public-methods", - "too-many-return-statements", - "too-many-statements", - "unexpected-keyword-arg", - "ungrouped-imports", - "unsubscriptable-object", - "unsupported-assignment-operation", - "unsupported-membership-test", - "unused-import", - "use-implicit-booleaness-not-comparison", - "use-implicit-booleaness-not-len", - "wrong-import-order", - "wrong-import-position", - - # misc - "abstract-class-instantiated", - "no-value-for-parameter", - "undefined-variable", - "unpacking-non-sequence", - - # pylint type "C": convention, for programming standard violation - "missing-class-docstring", - "missing-function-docstring", - "missing-module-docstring", - "too-many-lines", - "unidiomatic-typecheck", - "unnecessary-dunder-call", - "unnecessary-lambda-assignment", - - # pylint type "R": refactor, for bad code smell - "consider-using-with", - "cyclic-import", - "duplicate-code", - "inconsistent-return-statements", - "redefined-argument-from-local", - "too-few-public-methods", - - # pylint type "W": warning, for python specific problems - "abstract-method", - "arguments-differ", - "arguments-out-of-order", - "arguments-renamed", - "attribute-defined-outside-init", - "comparison-with-callable", - "dangerous-default-value", - "deprecated-module", - "eval-used", - "expression-not-assigned", - "fixme", - "global-statement", - "invalid-overridden-method", - "keyword-arg-before-vararg", - "possibly-unused-variable", - "protected-access", - "raise-missing-from", - "redefined-builtin", - "redefined-outer-name", - "self-cls-assignment", - "signature-differs", - "super-init-not-called", - "try-except-raise", - "unnecessary-lambda", - "unspecified-encoding", - "unused-argument", - "unused-variable", - "using-constant-test" -] - -[tool.pytest.ini_options] -# sync minversion with pyproject.toml & install.rst -minversion = "7.0" -addopts = "--strict-data-files --strict-markers --strict-config --capture=no --durations=30 --junitxml=test-data.xml" -empty_parameter_set_mark = "fail_at_collect" -xfail_strict = true -testpaths = "pandas" -doctest_optionflags = [ - "NORMALIZE_WHITESPACE", - "IGNORE_EXCEPTION_DETAIL", - "ELLIPSIS", -] -filterwarnings = [ - # Will be fixed in numba 0.56: https://github.com/numba/numba/issues/7758 - "ignore:`np.MachAr` is deprecated:DeprecationWarning:numba", - "ignore:.*urllib3:DeprecationWarning:botocore", - "ignore:Setuptools is replacing distutils.:UserWarning:_distutils_hack", - # https://github.com/PyTables/PyTables/issues/822 - "ignore:a closed node found in the registry:UserWarning:tables", - "ignore:`np.object` is a deprecated:DeprecationWarning:tables", - "ignore:tostring:DeprecationWarning:tables", - "ignore:distutils Version classes are deprecated:DeprecationWarning:numexpr", - "ignore:distutils Version classes are deprecated:DeprecationWarning:fastparquet", - "ignore:distutils Version classes are deprecated:DeprecationWarning:fsspec", -] -junit_family = "xunit2" -markers = [ - "single_cpu: tests that should run on a single cpu only", - "slow: mark a test as slow", - "network: mark a test as network", - "db: tests requiring a database (mysql or postgres)", - "clipboard: mark a pd.read_clipboard test", - "arm_slow: mark a test as slow for arm64 architecture", - "arraymanager: mark a test to run with ArrayManager enabled", -] -asyncio_mode = "strict" - -[tool.mypy] -# Import discovery -mypy_path = "typings" -files = ["pandas", "typings"] -namespace_packages = false -explicit_package_bases = false -ignore_missing_imports = true -follow_imports = "normal" -follow_imports_for_stubs = false -no_site_packages = false -no_silence_site_packages = false -# Platform configuration -python_version = "3.8" -platform = "linux-64" -# Disallow dynamic typing -disallow_any_unimported = false # TODO -disallow_any_expr = false # TODO -disallow_any_decorated = false # TODO -disallow_any_explicit = false # TODO -disallow_any_generics = false # TODO -disallow_subclassing_any = false # TODO -# Untyped definitions and calls -disallow_untyped_calls = false # TODO -disallow_untyped_defs = false # TODO -disallow_incomplete_defs = false # TODO -check_untyped_defs = true -disallow_untyped_decorators = true -# None and Optional handling -no_implicit_optional = true -strict_optional = true -# Configuring warnings -warn_redundant_casts = true -warn_unused_ignores = true -warn_no_return = true -warn_return_any = false # TODO -warn_unreachable = false # GH#27396 -# Suppressing errors -ignore_errors = false -enable_error_code = "ignore-without-code" -# Miscellaneous strictness flags -allow_untyped_globals = false -allow_redefinition = false -local_partial_types = false -implicit_reexport = true -strict_equality = true -# Configuring error messages -show_error_context = false -show_column_numbers = false -show_error_codes = true - -[[tool.mypy.overrides]] -module = [ - "pandas.tests.*", - "pandas._version", - "pandas.io.clipboard", -] -check_untyped_defs = false - -[[tool.mypy.overrides]] -module = [ - "pandas.tests.apply.test_series_apply", - "pandas.tests.arithmetic.conftest", - "pandas.tests.arrays.sparse.test_combine_concat", - "pandas.tests.dtypes.test_common", - "pandas.tests.frame.methods.test_to_records", - "pandas.tests.groupby.test_rank", - "pandas.tests.groupby.transform.test_transform", - "pandas.tests.indexes.interval.test_interval", - "pandas.tests.indexing.test_categorical", - "pandas.tests.io.excel.test_writers", - "pandas.tests.reductions.test_reductions", - "pandas.tests.test_expressions", -] -ignore_errors = true - -# To be kept consistent with "Import Formatting" section in contributing.rst -[tool.isort] -known_pre_libs = "pandas._config" -known_pre_core = ["pandas._libs", "pandas._typing", "pandas.util._*", "pandas.compat", "pandas.errors"] -known_dtypes = "pandas.core.dtypes" -known_post_core = ["pandas.tseries", "pandas.io", "pandas.plotting"] -sections = ["FUTURE", "STDLIB", "THIRDPARTY" ,"PRE_LIBS" , "PRE_CORE", "DTYPES", "FIRSTPARTY", "POST_CORE", "LOCALFOLDER"] -profile = "black" -combine_as_imports = true -force_grid_wrap = 2 -force_sort_within_sections = true -skip_glob = "env" -skip = "pandas/__init__.py" - -[tool.pyright] -pythonVersion = "3.8" -typeCheckingMode = "basic" -include = ["pandas", "typings"] -exclude = ["pandas/tests", "pandas/io/clipboard", "pandas/util/version"] -# enable subset of "strict" -reportDuplicateImport = true -reportInvalidStubStatement = true -reportOverlappingOverload = true -reportPropertyTypeMismatch = true -reportUntypedClassDecorator = true -reportUntypedFunctionDecorator = true -reportUntypedNamedTuple = true -reportUnusedImport = true -# disable subset of "basic" -reportGeneralTypeIssues = false -reportMissingModuleSource = false -reportOptionalCall = false -reportOptionalIterable = false -reportOptionalMemberAccess = false -reportOptionalOperand = false -reportOptionalSubscript = false -reportPrivateImportUsage = false -reportUnboundVariable = false - -[tool.coverage.run] -branch = true -omit = ["pandas/_typing.py", "pandas/_version.py"] -plugins = ["Cython.Coverage"] -source = ["pandas"] - -[tool.coverage.report] -ignore_errors = false -show_missing = true -omit = ["pandas/_version.py"] -exclude_lines = [ - # Have to re-enable the standard pragma - "pragma: no cover", - # Don't complain about missing debug-only code:s - "def __repr__", - "if self.debug", - # Don't complain if tests don't hit defensive assertion code: - "raise AssertionError", - "raise NotImplementedError", - "AbstractMethodError", - # Don't complain if non-runnable code isn't run: - "if 0:", - "if __name__ == .__main__.:", - "if TYPE_CHECKING:", -] - -[tool.coverage.html] -directory = "coverage_html_report" - -[tool.codespell] -ignore-words-list = "blocs, coo, hist, nd, sav, ser, recuse" -ignore-regex = 'https://([\w/\.])+' diff --git a/scripts/tests/dummy_yaml_expected_file_1.yaml b/scripts/tests/dummy_yaml_expected_file_1.yaml deleted file mode 100644 index e55edee5a5b82..0000000000000 --- a/scripts/tests/dummy_yaml_expected_file_1.yaml +++ /dev/null @@ -1,56 +0,0 @@ -name: pandas-dev -channels: - - conda-forge -dependencies: - - python=3.8 - - # build dependencies - - versioneer[toml] - - cython>=0.29.32 - - # test dependencies - - pytest>=7.0.0 - - pytest-cov - - pytest-xdist>=2.2.0 - - psutil - - pytest-asyncio>=0.17 - - boto3 - - # required dependencies - - python-dateutil - - numpy - - pytz - - # optional dependencies - - beautifulsoup4>=5.9.3 - - blosc - - bottleneck>=1.3.2 - - brotlipy>=0.7.0 - - fastparquet>=0.6.3 - - fsspec>=2021.07.0 - - html5lib>=1.1 - - hypothesis>=6.34.2 - - gcsfs>=2021.07.0 - - jinja2>=3.0.0 - - lxml>=4.6.3 - - matplotlib>=3.6.1 - - numba>=0.53.1 - - numexpr>=2.7.3 - - openpyxl>=3.0.7 - - odfpy>=1.4.1 - - pandas-gbq>=0.15.0 - - psycopg2>=2.8.6 - - pyarrow<11, >=7.0.0 - - pymysql>=1.0.2 - - pyreadstat>=1.1.2 - - pytables>=3.6.1 - - python-snappy>=0.6.0 - - pyxlsb>=1.0.8 - - s3fs>=2021.08.0 - - scipy>=1.7.1 - - sqlalchemy>=1.4.16 - - tabulate>=0.8.9 - - xarray>=0.21.0 - - xlrd>=2.0.1 - - xlsxwriter>=1.4.3 - - zstandard>=0.15.2 diff --git a/scripts/tests/dummy_yaml_unmodified_file_1.yaml b/scripts/tests/dummy_yaml_unmodified_file_1.yaml deleted file mode 100644 index 45c4b332c74e8..0000000000000 --- a/scripts/tests/dummy_yaml_unmodified_file_1.yaml +++ /dev/null @@ -1,56 +0,0 @@ -name: pandas-dev -channels: - - conda-forge -dependencies: - - python=3.8 - - # build dependencies - - versioneer[toml] - - cython>=0.29.32 - - # test dependencies - - pytest>=7.0.0 - - pytest-cov - - pytest-xdist>=2.2.0 - - psutil - - pytest-asyncio>=0.17 - - boto3 - - # required dependencies - - python-dateutil - - numpy - - pytz - - # optional dependencies - - beautifulsoup4 - - blosc - - bottleneck>=1.3.2 - - brotlipy - - fastparquet>=0.6.3 - - fsspec>=2021.07.0 - - html5lib>=1.1 - - hypothesis - - gcsfs>=2021.07.0 - - jinja2 - - lxml>=4.6.3 - - matplotlib>=3.6.1 - - numba - - numexpr>=2.7.3 - - openpyxl>=3.0.7 - - odfpy>=1.4.1 - - pandas-gbq>=0.15.0 - - psycopg2 - - pyarrow<11, >=7.0.0 - - pymysql>=1.0.2 - - pyreadstat>=1.1.2 - - pytables>=3.6.1 - - python-snappy>=0.6.0 - - pyxlsb>=1.0.8 - - s3fs>=2021.08.0 - - scipy>=1.7.1 - - sqlalchemy>=1.4.16 - - tabulate>=0.8.9 - - xarray>=0.21.0 - - xlrd>=2.0.1 - - xlsxwriter>=1.4.3 - - zstandard>=0.15.2 From 3b6123ffaa44341a7bc3ceec49bddfb663e082a6 Mon Sep 17 00:00:00 2001 From: doge Date: Wed, 15 Feb 2023 12:27:06 -0800 Subject: [PATCH 19/33] Fix tzdata bug --- scripts/validate_min_versions_in_sync.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/scripts/validate_min_versions_in_sync.py b/scripts/validate_min_versions_in_sync.py index 9cc4293cfc499..d8807e93083aa 100755 --- a/scripts/validate_min_versions_in_sync.py +++ b/scripts/validate_min_versions_in_sync.py @@ -101,7 +101,10 @@ def update_yaml_file_version(yaml_file, toml_dic): if version.parse(yaml_version) <= version.parse( toml_deps[yaml_package] ): - yaml_left = yaml_package + if "tzdata" in yaml_package: + yaml_left += operator + else: + yaml_left = yaml_package else: yaml_left = str(dep_line) + ", " else: From 4af5291a272d15fad9dacd42ab5a545eb1265728 Mon Sep 17 00:00:00 2001 From: doge Date: Wed, 15 Feb 2023 12:32:42 -0800 Subject: [PATCH 20/33] Fix tzdata issue --- scripts/validate_min_versions_in_sync.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/validate_min_versions_in_sync.py b/scripts/validate_min_versions_in_sync.py index d8807e93083aa..ba39d63c0865c 100755 --- a/scripts/validate_min_versions_in_sync.py +++ b/scripts/validate_min_versions_in_sync.py @@ -102,7 +102,7 @@ def update_yaml_file_version(yaml_file, toml_dic): toml_deps[yaml_package] ): if "tzdata" in yaml_package: - yaml_left += operator + yaml_left = yaml_package + operator else: yaml_left = yaml_package else: From 5bd4908cb135a809ecaacb68fc3c9b328295f6a3 Mon Sep 17 00:00:00 2001 From: doge Date: Sun, 19 Feb 2023 20:29:16 -0800 Subject: [PATCH 21/33] Refactor --- environment.yml | 2 +- requirements-dev.txt | 2 +- .../dummy_yaml_expected_file_1.yaml | 2 +- .../test_validate_min_versions_in_sync.py | 31 +++- scripts/validate_min_versions_in_sync.py | 174 +++++++++++------- 5 files changed, 137 insertions(+), 74 deletions(-) diff --git a/environment.yml b/environment.yml index bbef9a0db7ac9..21d2635d0420a 100644 --- a/environment.yml +++ b/environment.yml @@ -15,7 +15,7 @@ dependencies: - pytest-cov - pytest-xdist>=2.2.0 - psutil - - pytest-asyncio>=0.17 + - pytest-asyncio>=0.17.0 - coverage # required dependencies diff --git a/requirements-dev.txt b/requirements-dev.txt index 04a2a54ea2c43..00ef930d40d28 100644 --- a/requirements-dev.txt +++ b/requirements-dev.txt @@ -8,7 +8,7 @@ pytest>=7.0.0 pytest-cov pytest-xdist>=2.2.0 psutil -pytest-asyncio>=0.17 +pytest-asyncio>=0.17.0 coverage python-dateutil numpy diff --git a/scripts/tests/dummy_test_files/dummy_yaml_expected_file_1.yaml b/scripts/tests/dummy_test_files/dummy_yaml_expected_file_1.yaml index 35e5c5eed76fa..be5e467b57e10 100644 --- a/scripts/tests/dummy_test_files/dummy_yaml_expected_file_1.yaml +++ b/scripts/tests/dummy_test_files/dummy_yaml_expected_file_1.yaml @@ -14,7 +14,7 @@ dependencies: - pytest-cov - pytest-xdist>=2.2.0 - psutil - - pytest-asyncio>=0.17 + - pytest-asyncio>=0.17.0 - boto3 # required dependencies diff --git a/scripts/tests/test_validate_min_versions_in_sync.py b/scripts/tests/test_validate_min_versions_in_sync.py index 9556708175308..e4cc0c9235a0f 100644 --- a/scripts/tests/test_validate_min_versions_in_sync.py +++ b/scripts/tests/test_validate_min_versions_in_sync.py @@ -2,13 +2,18 @@ import sys import pytest +import yaml if sys.version_info >= (3, 11): import tomllib else: import tomli as tomllib -from scripts.validate_min_versions_in_sync import update_yaml_file_version +from scripts.validate_min_versions_in_sync import ( + get_toml_map_from, + get_yaml_map_from, + pin_min_versions_to_yaml_file, +) @pytest.mark.parametrize( @@ -50,7 +55,7 @@ "scripts/tests/dummy_test_files/dummy_yaml_expected_file_4.yaml" ), ), - ( # range version + ( # range pathlib.Path("scripts/tests/dummy_test_files/dummy_toml_file.toml"), pathlib.Path( "scripts/tests/dummy_test_files/dummy_yaml_unmodified_file_5.yaml" @@ -61,11 +66,27 @@ ), ], ) -def test_update_yaml_file_version(src_toml, src_yaml, expected_yaml): +def test_pin_min_versions_to_yaml_file(src_toml, src_yaml, expected_yaml): with open(src_toml, "rb") as toml_f: - toml_dic = tomllib.load(toml_f) + toml_map = tomllib.load(toml_f) with open(src_yaml) as yaml_f: - result_yaml_file = update_yaml_file_version(yaml_f, toml_dic) + yaml_file_data = yaml_f.read() + yaml_file = yaml.safe_load(yaml_file_data) + yaml_dependencies = yaml_file["dependencies"] + yaml_map = get_yaml_map_from(yaml_dependencies) + toml_map = get_toml_map_from(toml_map) + print("SOURCE", yaml_file_data) + result_yaml_file = pin_min_versions_to_yaml_file( + yaml_map, toml_map, yaml_file_data + ) + print("RESULT", result_yaml_file) with open(expected_yaml) as yaml_f: dummy_yaml_expected_file_1 = yaml_f.read() assert result_yaml_file == dummy_yaml_expected_file_1 + + +test_pin_min_versions_to_yaml_file( + pathlib.Path("scripts/tests/dummy_test_files/dummy_toml_file.toml"), + pathlib.Path("scripts/tests/dummy_test_files/dummy_yaml_unmodified_file_1.yaml"), + pathlib.Path("scripts/tests/dummy_test_files/dummy_yaml_expected_file_1.yaml"), +) diff --git a/scripts/validate_min_versions_in_sync.py b/scripts/validate_min_versions_in_sync.py index ba39d63c0865c..f9268e43de62a 100755 --- a/scripts/validate_min_versions_in_sync.py +++ b/scripts/validate_min_versions_in_sync.py @@ -34,6 +34,10 @@ YAML_PATH = pathlib.Path("ci/deps") ENV_PATH = pathlib.Path("environment.yml") EXCLUDE_DEPS = {"tzdata", "blosc"} +EXCLUSION_LIST = { + "python=3.8[build=*_pypy]": None, + "tzdata": None, +} # pandas package is not available # in pre-commit environment sys.path.append("pandas/compat") @@ -48,87 +52,125 @@ def pin_min_versions_to_ci_deps(): all_yaml_files = list(YAML_PATH.iterdir()) + all_yaml_files = list() all_yaml_files.append(ENV_PATH) + toml_dependencies = {} with open(SETUP_PATH, "rb") as toml_f: - toml_dic = tomllib.load(toml_f) + toml_dependencies = tomllib.load(toml_f) for curr_file in all_yaml_files: with open(curr_file) as yaml_f: - data = update_yaml_file_version(yaml_f, toml_dic) + yaml_file_data = yaml_f.read() + yaml_file = yaml.safe_load(yaml_file_data) + yaml_dependencies = yaml_file["dependencies"] + res = [] + [res.append(x) for x in yaml_dependencies if x not in res] + yaml_dependencies = res + yaml_map = get_yaml_map_from(yaml_dependencies) + toml_map = get_toml_map_from(toml_dependencies) + yaml_file_data = pin_min_versions_to_yaml_file( + yaml_map, toml_map, yaml_file_data + ) os.remove(curr_file) with open(curr_file, "w") as f: - f.write(data) + f.write(yaml_file_data) -def update_yaml_file_version(yaml_file, toml_dic): - exclusion_list = { - "python=3.8[build=*_pypy]": None, - } +def get_toml_map_from(toml_dic) -> dict[str, str]: toml_deps = {} toml_dependencies = set(toml_dic["project"]["optional-dependencies"]["all"]) for dependency in toml_dependencies: toml_package, toml_version = dependency.strip().split(">=") toml_deps[toml_package] = toml_version - data = yaml_file.read() - yaml_f = yaml.safe_load(data) - yaml_deps = yaml_f["dependencies"] - res = [] - [res.append(x) for x in yaml_deps if x not in res] - yaml_deps = res - for dep_line in yaml_deps: - replace_text = operator = yaml_left = "" - search_text = str(dep_line) - if str(dep_line) in exclusion_list: + return toml_deps + + +def get_operator_from(dependency) -> str: + if "<=" in dependency: + operator = "<=" + elif ">=" in dependency: + operator = ">=" + elif "=" in dependency: + operator = "=" + elif ">" in dependency: + operator = ">" + elif "<" in dependency: + operator = "<" + else: + operator = None + return operator + + +def get_yaml_map_from(yaml_dic) -> dict[str, str]: + yaml_map = {} + for dependency in yaml_dic: + if type(dependency) == dict: continue - if ">=" in dep_line: - operator = ">=" - elif "==" in dep_line: - operator = "==" - elif "=" in dep_line: - operator = "=" - elif "<" in dep_line: - operator = "<" - elif ">" in dep_line: - operator = ">" - else: - operator = "" - if operator == "": - yaml_package, yaml_version = str(dep_line).strip(), None - yaml_left = yaml_package + search_text = str(dependency) + operator = get_operator_from(search_text) + if dependency in EXCLUSION_LIST: + continue + if "," in dependency: + yaml_dependency, yaml_version1 = search_text.split(",") + operator = get_operator_from(yaml_dependency) + yaml_package, yaml_version2 = yaml_dependency.split(operator) + yaml_version2 = operator + yaml_version2 + yaml_map[yaml_package] = [yaml_version1, yaml_version2] + elif ( + operator == ">=" + or operator == "<=" + or operator == ">" + or operator == "<" + or operator == "=" + ): + yaml_package, yaml_version = search_text.split(operator) + yaml_version = operator + yaml_version + yaml_map[yaml_package] = [yaml_version] else: - yaml_package, yaml_version = str(dep_line).strip().split(operator) - if "<" in operator or ">" in operator: - if yaml_package in toml_deps: - if version.parse(yaml_version) <= version.parse( - toml_deps[yaml_package] - ): - if "tzdata" in yaml_package: - yaml_left = yaml_package + operator - else: - yaml_left = yaml_package - else: - yaml_left = str(dep_line) + ", " - else: - yaml_left = yaml_package + operator - if yaml_package in toml_deps: - if ">" in yaml_left or "<" in yaml_left: - if "," in yaml_left: - # ex: "numpy<1.24.0," + ">=" + "1.2" - replace_text = yaml_left + ">=" + toml_deps[yaml_package] - else: - replace_text = yaml_left + toml_deps[yaml_package] - # update yaml package version to TOML min version - elif yaml_version is not None: - if version.parse(toml_deps[yaml_package]) > version.parse(yaml_version): - # ex: "hypothesis>=" + "6.34.2" - replace_text = yaml_left + toml_deps[yaml_package] - elif version.parse(toml_deps[yaml_package]) == version.parse( - yaml_version - ): - replace_text = dep_line - else: - # ex: "hypothesis + ">=" + 6.34.2" - replace_text = yaml_package + ">=" + toml_deps[yaml_package] - data = data.replace(search_text, replace_text) + yaml_package, yaml_version = search_text.strip(), None + yaml_map[yaml_package] = yaml_version + return yaml_map + + +def clean_version_list(yaml_versions, toml_version): + for i in range(len(yaml_versions)): + yaml_version = yaml_versions[i] + operator = get_operator_from(yaml_version) + if "<=" in operator or ">=" in operator: + yaml_version = yaml_version[2:] + elif "<" in operator or ">" in operator or "=" in operator: + yaml_version = yaml_version[1:] + yaml_version = version.parse(yaml_version) + if yaml_version < toml_version: + yaml_versions[i] = "-" + str(yaml_version) + elif yaml_version >= toml_version: + if ">" in operator or "=" in operator: + yaml_versions[i] = "-" + str(yaml_version) + return yaml_versions + + +def pin_min_versions_to_yaml_file(yaml_map, toml_map, yaml_file_data): + data = yaml_file_data + for yaml_pkg, yaml_versions in yaml_map.items(): + old_dep = yaml_pkg + if yaml_versions is not None: + for yaml_version in yaml_versions: + old_dep += yaml_version + ", " + old_dep = old_dep[:-2] + if yaml_pkg in toml_map: + min_dep = toml_map[yaml_pkg] + if yaml_versions is None: + new_dep = old_dep + ">=" + min_dep + data = data.replace(old_dep, new_dep, 1) + continue + toml_version = version.parse(min_dep) + yaml_versions = clean_version_list(yaml_versions, toml_version) + cleaned_yaml_versions = [] + [cleaned_yaml_versions.append(x) for x in yaml_versions if "-" not in x] + new_dep = yaml_pkg + for yaml_version in cleaned_yaml_versions: + new_dep += yaml_version + ", " + new_dep += ">=" + min_dep + data = data.replace(old_dep, new_dep) return data From 04dda2e027ff0c9b3a11da8c702f74ea735608a7 Mon Sep 17 00:00:00 2001 From: doge Date: Sun, 19 Feb 2023 21:09:10 -0800 Subject: [PATCH 22/33] Update ci deps --- ci/deps/actions-310-numpydev.yaml | 2 +- ci/deps/actions-310.yaml | 2 +- ci/deps/actions-311.yaml | 2 +- ci/deps/actions-38-downstream_compat.yaml | 2 +- ci/deps/actions-38-minimum_versions.yaml | 2 +- ci/deps/actions-38.yaml | 2 +- ci/deps/actions-39.yaml | 2 +- ci/deps/circle-38-arm64.yaml | 2 +- .../test_validate_min_versions_in_sync.py | 9 ----- scripts/validate_min_versions_in_sync.py | 35 ++++++++----------- 10 files changed, 23 insertions(+), 37 deletions(-) diff --git a/ci/deps/actions-310-numpydev.yaml b/ci/deps/actions-310-numpydev.yaml index d1c4338f1806e..1a461319685d2 100644 --- a/ci/deps/actions-310-numpydev.yaml +++ b/ci/deps/actions-310-numpydev.yaml @@ -12,7 +12,7 @@ dependencies: - pytest-cov - pytest-xdist>=2.2.0 - hypothesis>=6.34.2 - - pytest-asyncio>=0.17 + - pytest-asyncio>=0.17.0 # pandas dependencies - python-dateutil diff --git a/ci/deps/actions-310.yaml b/ci/deps/actions-310.yaml index 67cdd68aa616e..9839000c41607 100644 --- a/ci/deps/actions-310.yaml +++ b/ci/deps/actions-310.yaml @@ -13,7 +13,7 @@ dependencies: - pytest-cov - pytest-xdist>=2.2.0 - psutil - - pytest-asyncio>=0.17 + - pytest-asyncio>=0.17.0 - boto3 # required dependencies diff --git a/ci/deps/actions-311.yaml b/ci/deps/actions-311.yaml index 04f8cd1b8289b..a3300447813f7 100644 --- a/ci/deps/actions-311.yaml +++ b/ci/deps/actions-311.yaml @@ -13,7 +13,7 @@ dependencies: - pytest-cov - pytest-xdist>=2.2.0 - psutil - - pytest-asyncio>=0.17 + - pytest-asyncio>=0.17.0 - boto3 # required dependencies diff --git a/ci/deps/actions-38-downstream_compat.yaml b/ci/deps/actions-38-downstream_compat.yaml index c62aadfcce298..ffac34a47d124 100644 --- a/ci/deps/actions-38-downstream_compat.yaml +++ b/ci/deps/actions-38-downstream_compat.yaml @@ -14,7 +14,7 @@ dependencies: - pytest-cov - pytest-xdist>=2.2.0 - psutil - - pytest-asyncio>=0.17 + - pytest-asyncio>=0.17.0 - boto3 # required dependencies diff --git a/ci/deps/actions-38-minimum_versions.yaml b/ci/deps/actions-38-minimum_versions.yaml index 706549d6af0cb..88e5569e31c1a 100644 --- a/ci/deps/actions-38-minimum_versions.yaml +++ b/ci/deps/actions-38-minimum_versions.yaml @@ -15,7 +15,7 @@ dependencies: - pytest-cov - pytest-xdist>=2.2.0 - psutil - - pytest-asyncio>=0.17 + - pytest-asyncio>=0.17.0 - boto3 # required dependencies diff --git a/ci/deps/actions-38.yaml b/ci/deps/actions-38.yaml index b06fc18bd791b..f57a09f9d3a6c 100644 --- a/ci/deps/actions-38.yaml +++ b/ci/deps/actions-38.yaml @@ -13,7 +13,7 @@ dependencies: - pytest-cov - pytest-xdist>=2.2.0 - psutil - - pytest-asyncio>=0.17 + - pytest-asyncio>=0.17.0 - boto3 # required dependencies diff --git a/ci/deps/actions-39.yaml b/ci/deps/actions-39.yaml index a48186d346583..3f1b6dd1182de 100644 --- a/ci/deps/actions-39.yaml +++ b/ci/deps/actions-39.yaml @@ -13,7 +13,7 @@ dependencies: - pytest-cov - pytest-xdist>=2.2.0 - psutil - - pytest-asyncio>=0.17 + - pytest-asyncio>=0.17.0 - boto3 # required dependencies diff --git a/ci/deps/circle-38-arm64.yaml b/ci/deps/circle-38-arm64.yaml index bb9edb6284392..08160ac6f5806 100644 --- a/ci/deps/circle-38-arm64.yaml +++ b/ci/deps/circle-38-arm64.yaml @@ -13,7 +13,7 @@ dependencies: - pytest-cov - pytest-xdist>=2.2.0 - psutil - - pytest-asyncio>=0.17 + - pytest-asyncio>=0.17.0 - boto3 # required dependencies diff --git a/scripts/tests/test_validate_min_versions_in_sync.py b/scripts/tests/test_validate_min_versions_in_sync.py index e4cc0c9235a0f..ae1613b967b8c 100644 --- a/scripts/tests/test_validate_min_versions_in_sync.py +++ b/scripts/tests/test_validate_min_versions_in_sync.py @@ -75,18 +75,9 @@ def test_pin_min_versions_to_yaml_file(src_toml, src_yaml, expected_yaml): yaml_dependencies = yaml_file["dependencies"] yaml_map = get_yaml_map_from(yaml_dependencies) toml_map = get_toml_map_from(toml_map) - print("SOURCE", yaml_file_data) result_yaml_file = pin_min_versions_to_yaml_file( yaml_map, toml_map, yaml_file_data ) - print("RESULT", result_yaml_file) with open(expected_yaml) as yaml_f: dummy_yaml_expected_file_1 = yaml_f.read() assert result_yaml_file == dummy_yaml_expected_file_1 - - -test_pin_min_versions_to_yaml_file( - pathlib.Path("scripts/tests/dummy_test_files/dummy_toml_file.toml"), - pathlib.Path("scripts/tests/dummy_test_files/dummy_yaml_unmodified_file_1.yaml"), - pathlib.Path("scripts/tests/dummy_test_files/dummy_yaml_expected_file_1.yaml"), -) diff --git a/scripts/validate_min_versions_in_sync.py b/scripts/validate_min_versions_in_sync.py index f9268e43de62a..01a9ed573bba2 100755 --- a/scripts/validate_min_versions_in_sync.py +++ b/scripts/validate_min_versions_in_sync.py @@ -52,7 +52,6 @@ def pin_min_versions_to_ci_deps(): all_yaml_files = list(YAML_PATH.iterdir()) - all_yaml_files = list() all_yaml_files.append(ENV_PATH) toml_dependencies = {} with open(SETUP_PATH, "rb") as toml_f: @@ -103,25 +102,17 @@ def get_operator_from(dependency) -> str: def get_yaml_map_from(yaml_dic) -> dict[str, str]: yaml_map = {} for dependency in yaml_dic: - if type(dependency) == dict: + if type(dependency) == dict or dependency in EXCLUSION_LIST: continue search_text = str(dependency) operator = get_operator_from(search_text) - if dependency in EXCLUSION_LIST: - continue if "," in dependency: yaml_dependency, yaml_version1 = search_text.split(",") operator = get_operator_from(yaml_dependency) yaml_package, yaml_version2 = yaml_dependency.split(operator) yaml_version2 = operator + yaml_version2 yaml_map[yaml_package] = [yaml_version1, yaml_version2] - elif ( - operator == ">=" - or operator == "<=" - or operator == ">" - or operator == "<" - or operator == "=" - ): + elif operator is not None: yaml_package, yaml_version = search_text.split(operator) yaml_version = operator + yaml_version yaml_map[yaml_package] = [yaml_version] @@ -137,39 +128,43 @@ def clean_version_list(yaml_versions, toml_version): operator = get_operator_from(yaml_version) if "<=" in operator or ">=" in operator: yaml_version = yaml_version[2:] - elif "<" in operator or ">" in operator or "=" in operator: + else: yaml_version = yaml_version[1:] yaml_version = version.parse(yaml_version) if yaml_version < toml_version: yaml_versions[i] = "-" + str(yaml_version) elif yaml_version >= toml_version: - if ">" in operator or "=" in operator: + if ">" in operator: yaml_versions[i] = "-" + str(yaml_version) return yaml_versions def pin_min_versions_to_yaml_file(yaml_map, toml_map, yaml_file_data): data = yaml_file_data - for yaml_pkg, yaml_versions in yaml_map.items(): - old_dep = yaml_pkg + for yaml_package, yaml_versions in yaml_map.items(): + old_dep = yaml_package if yaml_versions is not None: for yaml_version in yaml_versions: old_dep += yaml_version + ", " old_dep = old_dep[:-2] - if yaml_pkg in toml_map: - min_dep = toml_map[yaml_pkg] + if yaml_package in toml_map: + cleaned_yaml_versions = [] + min_dep = toml_map[yaml_package] if yaml_versions is None: new_dep = old_dep + ">=" + min_dep data = data.replace(old_dep, new_dep, 1) continue toml_version = version.parse(min_dep) yaml_versions = clean_version_list(yaml_versions, toml_version) - cleaned_yaml_versions = [] [cleaned_yaml_versions.append(x) for x in yaml_versions if "-" not in x] - new_dep = yaml_pkg + new_dep = yaml_package for yaml_version in cleaned_yaml_versions: new_dep += yaml_version + ", " - new_dep += ">=" + min_dep + operator = get_operator_from(new_dep) + if operator != "=": + new_dep += ">=" + min_dep + else: + new_dep = new_dep[:-2] data = data.replace(old_dep, new_dep) return data From db0feae3d4733b67d2e2eb0b64088a3a4426cb35 Mon Sep 17 00:00:00 2001 From: doge Date: Mon, 20 Feb 2023 05:28:09 -0800 Subject: [PATCH 23/33] Remove pyarrow --- ci/deps/actions-311.yaml | 2 +- ci/deps/actions-38-downstream_compat.yaml | 2 +- ci/deps/actions-38.yaml | 2 +- ci/deps/actions-39.yaml | 2 +- ci/deps/circle-38-arm64.yaml | 2 +- scripts/validate_min_versions_in_sync.py | 1 + 6 files changed, 6 insertions(+), 5 deletions(-) diff --git a/ci/deps/actions-311.yaml b/ci/deps/actions-311.yaml index a3300447813f7..1ab34e5944bbe 100644 --- a/ci/deps/actions-311.yaml +++ b/ci/deps/actions-311.yaml @@ -42,7 +42,7 @@ dependencies: - psycopg2>=2.8.6 - pymysql>=1.0.2 # - pytables>=3.8.0 # first version that supports 3.11 - - pyarrow>=7.0.0 + - pyarrow - pyreadstat>=1.1.2 - python-snappy>=0.6.0 - pyxlsb>=1.0.8 diff --git a/ci/deps/actions-38-downstream_compat.yaml b/ci/deps/actions-38-downstream_compat.yaml index ffac34a47d124..021bb6b13595e 100644 --- a/ci/deps/actions-38-downstream_compat.yaml +++ b/ci/deps/actions-38-downstream_compat.yaml @@ -40,7 +40,7 @@ dependencies: - openpyxl>=3.0.7, <3.1.1 - odfpy>=1.4.1 - psycopg2>=2.8.6 - - pyarrow<11, >=7.0.0 + - pyarrow - pymysql>=1.0.2 - pyreadstat>=1.1.2 - pytables>=3.6.1 diff --git a/ci/deps/actions-38.yaml b/ci/deps/actions-38.yaml index f57a09f9d3a6c..6a815b0a4f571 100644 --- a/ci/deps/actions-38.yaml +++ b/ci/deps/actions-38.yaml @@ -40,7 +40,7 @@ dependencies: - odfpy>=1.4.1 - pandas-gbq>=0.15.0 - psycopg2>=2.8.6 - - pyarrow<11, >=7.0.0 + - pyarrow - pymysql>=1.0.2 - pyreadstat>=1.1.2 - pytables>=3.6.1 diff --git a/ci/deps/actions-39.yaml b/ci/deps/actions-39.yaml index 3f1b6dd1182de..43efc384ca779 100644 --- a/ci/deps/actions-39.yaml +++ b/ci/deps/actions-39.yaml @@ -41,7 +41,7 @@ dependencies: - pandas-gbq>=0.15.0 - psycopg2>=2.8.6 - pymysql>=1.0.2 - - pyarrow>=7.0.0 + - pyarrow - pyreadstat>=1.1.2 - pytables>=3.6.1 - python-snappy>=0.6.0 diff --git a/ci/deps/circle-38-arm64.yaml b/ci/deps/circle-38-arm64.yaml index 08160ac6f5806..48bca83c3c511 100644 --- a/ci/deps/circle-38-arm64.yaml +++ b/ci/deps/circle-38-arm64.yaml @@ -40,7 +40,7 @@ dependencies: - odfpy>=1.4.1 - pandas-gbq>=0.15.0 - psycopg2>=2.8.6 - - pyarrow>=7.0.0 + - pyarrow - pymysql>=1.0.2 # Not provided on ARM #- pyreadstat diff --git a/scripts/validate_min_versions_in_sync.py b/scripts/validate_min_versions_in_sync.py index 01a9ed573bba2..f89676dddc5f6 100755 --- a/scripts/validate_min_versions_in_sync.py +++ b/scripts/validate_min_versions_in_sync.py @@ -37,6 +37,7 @@ EXCLUSION_LIST = { "python=3.8[build=*_pypy]": None, "tzdata": None, + "pyarrow": None, } # pandas package is not available # in pre-commit environment From 4bd6e7ad40403ad3aed90668a4aa17c8c99478f2 Mon Sep 17 00:00:00 2001 From: doge Date: Mon, 20 Feb 2023 05:59:10 -0800 Subject: [PATCH 24/33] Revert tzdata --- ci/deps/actions-310.yaml | 2 +- ci/deps/actions-311.yaml | 2 +- ci/deps/actions-38-minimum_versions.yaml | 2 +- ci/deps/actions-39.yaml | 2 +- environment.yml | 4 ++-- requirements-dev.txt | 2 +- scripts/validate_min_versions_in_sync.py | 2 ++ 7 files changed, 9 insertions(+), 7 deletions(-) diff --git a/ci/deps/actions-310.yaml b/ci/deps/actions-310.yaml index 9839000c41607..210beecb6e6f9 100644 --- a/ci/deps/actions-310.yaml +++ b/ci/deps/actions-310.yaml @@ -50,7 +50,7 @@ dependencies: - scipy>=1.7.1 - sqlalchemy<1.4.46, >=1.4.16 - tabulate>=0.8.9 - - tzdata>=2022.1 + - tzdata>=2022a - xarray>=0.21.0 - xlrd>=2.0.1 - xlsxwriter>=1.4.3 diff --git a/ci/deps/actions-311.yaml b/ci/deps/actions-311.yaml index 1ab34e5944bbe..6df4f05aa351a 100644 --- a/ci/deps/actions-311.yaml +++ b/ci/deps/actions-311.yaml @@ -50,7 +50,7 @@ dependencies: - scipy>=1.7.1 - sqlalchemy<1.4.46, >=1.4.16 - tabulate>=0.8.9 - - tzdata>=2022.1 + - tzdata>=2022a - xarray>=0.21.0 - xlrd>=2.0.1 - xlsxwriter>=1.4.3 diff --git a/ci/deps/actions-38-minimum_versions.yaml b/ci/deps/actions-38-minimum_versions.yaml index 88e5569e31c1a..8812a1e59cdd4 100644 --- a/ci/deps/actions-38-minimum_versions.yaml +++ b/ci/deps/actions-38-minimum_versions.yaml @@ -53,7 +53,7 @@ dependencies: - scipy=1.7.1 - sqlalchemy=1.4.16 - tabulate=0.8.9 - - tzdata=2022.1 + - tzdata=2022a - xarray=0.21.0 - xlrd=2.0.1 - xlsxwriter=1.4.3 diff --git a/ci/deps/actions-39.yaml b/ci/deps/actions-39.yaml index 43efc384ca779..5d92c7eff2a57 100644 --- a/ci/deps/actions-39.yaml +++ b/ci/deps/actions-39.yaml @@ -50,7 +50,7 @@ dependencies: - scipy>=1.7.1 - sqlalchemy<1.4.46, >=1.4.16 - tabulate>=0.8.9 - - tzdata>=2022.1 + - tzdata>=2022a - xarray>=0.21.0 - xlrd>=2.0.1 - xlsxwriter>=1.4.3 diff --git a/environment.yml b/environment.yml index 6b25e41db2d5a..efed52229f44b 100644 --- a/environment.yml +++ b/environment.yml @@ -43,7 +43,7 @@ dependencies: - odfpy>=1.4.1 - py - psycopg2>=2.8.6 - - pyarrow<11, >=6.0.0 + - pyarrow - pymysql>=1.0.2 - pyreadstat>=1.1.2 - pytables>=3.6.1 @@ -53,7 +53,7 @@ dependencies: - scipy>=1.7.1 - sqlalchemy>=1.4.16 - tabulate>=0.8.9 - - tzdata>=2022.1 + - tzdata>=2022a - xarray>=0.21.0 - xlrd>=2.0.1 - xlsxwriter>=1.4.3 diff --git a/requirements-dev.txt b/requirements-dev.txt index a5a2bce38bd6f..762eaa26c1720 100644 --- a/requirements-dev.txt +++ b/requirements-dev.txt @@ -32,7 +32,7 @@ openpyxl>=3.0.7, <3.1.1 odfpy>=1.4.1 py psycopg2-binary>=2.8.6 -pyarrow<11, >=6.0.0 +pyarrow pymysql>=1.0.2 pyreadstat>=1.1.2 tables>=3.6.1 diff --git a/scripts/validate_min_versions_in_sync.py b/scripts/validate_min_versions_in_sync.py index f89676dddc5f6..b1dc734994b69 100755 --- a/scripts/validate_min_versions_in_sync.py +++ b/scripts/validate_min_versions_in_sync.py @@ -143,6 +143,8 @@ def clean_version_list(yaml_versions, toml_version): def pin_min_versions_to_yaml_file(yaml_map, toml_map, yaml_file_data): data = yaml_file_data for yaml_package, yaml_versions in yaml_map.items(): + if yaml_package in EXCLUSION_LIST: + continue old_dep = yaml_package if yaml_versions is not None: for yaml_version in yaml_versions: From bec1dd188338361e211eb048f940ac12d25e7e45 Mon Sep 17 00:00:00 2001 From: doge Date: Mon, 20 Feb 2023 09:02:08 -0800 Subject: [PATCH 25/33] Update pyarrow --- ci/deps/actions-311.yaml | 2 +- ci/deps/actions-38-downstream_compat.yaml | 2 +- ci/deps/actions-38.yaml | 2 +- ci/deps/actions-39.yaml | 2 +- ci/deps/circle-38-arm64.yaml | 2 +- environment.yml | 2 +- requirements-dev.txt | 2 +- scripts/tests/dummy_test_files/dummy_yaml_expected_file_5.yaml | 1 - .../tests/dummy_test_files/dummy_yaml_unmodified_file_5.yaml | 1 - scripts/validate_min_versions_in_sync.py | 1 - 10 files changed, 7 insertions(+), 10 deletions(-) diff --git a/ci/deps/actions-311.yaml b/ci/deps/actions-311.yaml index 6df4f05aa351a..b0bdfc7532471 100644 --- a/ci/deps/actions-311.yaml +++ b/ci/deps/actions-311.yaml @@ -42,7 +42,7 @@ dependencies: - psycopg2>=2.8.6 - pymysql>=1.0.2 # - pytables>=3.8.0 # first version that supports 3.11 - - pyarrow + - pyarrow>=7.0.0 - pyreadstat>=1.1.2 - python-snappy>=0.6.0 - pyxlsb>=1.0.8 diff --git a/ci/deps/actions-38-downstream_compat.yaml b/ci/deps/actions-38-downstream_compat.yaml index 021bb6b13595e..7ba2331bc4363 100644 --- a/ci/deps/actions-38-downstream_compat.yaml +++ b/ci/deps/actions-38-downstream_compat.yaml @@ -40,7 +40,7 @@ dependencies: - openpyxl>=3.0.7, <3.1.1 - odfpy>=1.4.1 - psycopg2>=2.8.6 - - pyarrow + - pyarrow>=7.0.0 - pymysql>=1.0.2 - pyreadstat>=1.1.2 - pytables>=3.6.1 diff --git a/ci/deps/actions-38.yaml b/ci/deps/actions-38.yaml index 6a815b0a4f571..0b761a783c65f 100644 --- a/ci/deps/actions-38.yaml +++ b/ci/deps/actions-38.yaml @@ -40,7 +40,7 @@ dependencies: - odfpy>=1.4.1 - pandas-gbq>=0.15.0 - psycopg2>=2.8.6 - - pyarrow + - pyarrow>=7.0.0 - pymysql>=1.0.2 - pyreadstat>=1.1.2 - pytables>=3.6.1 diff --git a/ci/deps/actions-39.yaml b/ci/deps/actions-39.yaml index 5d92c7eff2a57..9efd87ebcf074 100644 --- a/ci/deps/actions-39.yaml +++ b/ci/deps/actions-39.yaml @@ -41,7 +41,7 @@ dependencies: - pandas-gbq>=0.15.0 - psycopg2>=2.8.6 - pymysql>=1.0.2 - - pyarrow + - pyarrow>=7.0.0 - pyreadstat>=1.1.2 - pytables>=3.6.1 - python-snappy>=0.6.0 diff --git a/ci/deps/circle-38-arm64.yaml b/ci/deps/circle-38-arm64.yaml index 48bca83c3c511..08160ac6f5806 100644 --- a/ci/deps/circle-38-arm64.yaml +++ b/ci/deps/circle-38-arm64.yaml @@ -40,7 +40,7 @@ dependencies: - odfpy>=1.4.1 - pandas-gbq>=0.15.0 - psycopg2>=2.8.6 - - pyarrow + - pyarrow>=7.0.0 - pymysql>=1.0.2 # Not provided on ARM #- pyreadstat diff --git a/environment.yml b/environment.yml index efed52229f44b..64971a86fe8df 100644 --- a/environment.yml +++ b/environment.yml @@ -43,7 +43,7 @@ dependencies: - odfpy>=1.4.1 - py - psycopg2>=2.8.6 - - pyarrow + - pyarrow>=7.0.0 - pymysql>=1.0.2 - pyreadstat>=1.1.2 - pytables>=3.6.1 diff --git a/requirements-dev.txt b/requirements-dev.txt index 762eaa26c1720..c52abae23b84b 100644 --- a/requirements-dev.txt +++ b/requirements-dev.txt @@ -32,7 +32,7 @@ openpyxl>=3.0.7, <3.1.1 odfpy>=1.4.1 py psycopg2-binary>=2.8.6 -pyarrow +pyarrow>=7.0.0 pymysql>=1.0.2 pyreadstat>=1.1.2 tables>=3.6.1 diff --git a/scripts/tests/dummy_test_files/dummy_yaml_expected_file_5.yaml b/scripts/tests/dummy_test_files/dummy_yaml_expected_file_5.yaml index d8b437ef36c37..c8e25076ef3b0 100644 --- a/scripts/tests/dummy_test_files/dummy_yaml_expected_file_5.yaml +++ b/scripts/tests/dummy_test_files/dummy_yaml_expected_file_5.yaml @@ -3,4 +3,3 @@ dependencies: - jinja2<8, >=3.0.0 - scipy<9, >=1.7.1 - SQLAlchemy<2.0, >=1.4.16 - - pyarrow<11, >=7.0.0 diff --git a/scripts/tests/dummy_test_files/dummy_yaml_unmodified_file_5.yaml b/scripts/tests/dummy_test_files/dummy_yaml_unmodified_file_5.yaml index 97c013141d099..22882af2cbc4b 100644 --- a/scripts/tests/dummy_test_files/dummy_yaml_unmodified_file_5.yaml +++ b/scripts/tests/dummy_test_files/dummy_yaml_unmodified_file_5.yaml @@ -3,4 +3,3 @@ dependencies: - jinja2<8 - scipy<9 - SQLAlchemy<2.0 - - pyarrow<11 diff --git a/scripts/validate_min_versions_in_sync.py b/scripts/validate_min_versions_in_sync.py index b1dc734994b69..4b7e8c10dc9cc 100755 --- a/scripts/validate_min_versions_in_sync.py +++ b/scripts/validate_min_versions_in_sync.py @@ -37,7 +37,6 @@ EXCLUSION_LIST = { "python=3.8[build=*_pypy]": None, "tzdata": None, - "pyarrow": None, } # pandas package is not available # in pre-commit environment From 3c4e80086deaa92ace8ef878c52e63bb7c91876a Mon Sep 17 00:00:00 2001 From: doge Date: Mon, 20 Feb 2023 09:16:41 -0800 Subject: [PATCH 26/33] Revert pyarrow --- ci/deps/actions-310.yaml | 2 +- ci/deps/actions-311.yaml | 2 +- ci/deps/actions-38-downstream_compat.yaml | 2 +- ci/deps/actions-38.yaml | 2 +- ci/deps/actions-39.yaml | 2 +- ci/deps/circle-38-arm64.yaml | 2 +- scripts/validate_min_versions_in_sync.py | 1 + 7 files changed, 7 insertions(+), 6 deletions(-) diff --git a/ci/deps/actions-310.yaml b/ci/deps/actions-310.yaml index 210beecb6e6f9..5905a2f6a444d 100644 --- a/ci/deps/actions-310.yaml +++ b/ci/deps/actions-310.yaml @@ -42,7 +42,7 @@ dependencies: - psycopg2>=2.8.6 - pymysql>=1.0.2 - pytables>=3.6.1 - - pyarrow>=7.0.0 + - pyarrow - pyreadstat>=1.1.2 - python-snappy>=0.6.0 - pyxlsb>=1.0.8 diff --git a/ci/deps/actions-311.yaml b/ci/deps/actions-311.yaml index b0bdfc7532471..6df4f05aa351a 100644 --- a/ci/deps/actions-311.yaml +++ b/ci/deps/actions-311.yaml @@ -42,7 +42,7 @@ dependencies: - psycopg2>=2.8.6 - pymysql>=1.0.2 # - pytables>=3.8.0 # first version that supports 3.11 - - pyarrow>=7.0.0 + - pyarrow - pyreadstat>=1.1.2 - python-snappy>=0.6.0 - pyxlsb>=1.0.8 diff --git a/ci/deps/actions-38-downstream_compat.yaml b/ci/deps/actions-38-downstream_compat.yaml index 7ba2331bc4363..021bb6b13595e 100644 --- a/ci/deps/actions-38-downstream_compat.yaml +++ b/ci/deps/actions-38-downstream_compat.yaml @@ -40,7 +40,7 @@ dependencies: - openpyxl>=3.0.7, <3.1.1 - odfpy>=1.4.1 - psycopg2>=2.8.6 - - pyarrow>=7.0.0 + - pyarrow - pymysql>=1.0.2 - pyreadstat>=1.1.2 - pytables>=3.6.1 diff --git a/ci/deps/actions-38.yaml b/ci/deps/actions-38.yaml index 0b761a783c65f..6a815b0a4f571 100644 --- a/ci/deps/actions-38.yaml +++ b/ci/deps/actions-38.yaml @@ -40,7 +40,7 @@ dependencies: - odfpy>=1.4.1 - pandas-gbq>=0.15.0 - psycopg2>=2.8.6 - - pyarrow>=7.0.0 + - pyarrow - pymysql>=1.0.2 - pyreadstat>=1.1.2 - pytables>=3.6.1 diff --git a/ci/deps/actions-39.yaml b/ci/deps/actions-39.yaml index 9efd87ebcf074..5d92c7eff2a57 100644 --- a/ci/deps/actions-39.yaml +++ b/ci/deps/actions-39.yaml @@ -41,7 +41,7 @@ dependencies: - pandas-gbq>=0.15.0 - psycopg2>=2.8.6 - pymysql>=1.0.2 - - pyarrow>=7.0.0 + - pyarrow - pyreadstat>=1.1.2 - pytables>=3.6.1 - python-snappy>=0.6.0 diff --git a/ci/deps/circle-38-arm64.yaml b/ci/deps/circle-38-arm64.yaml index 08160ac6f5806..48bca83c3c511 100644 --- a/ci/deps/circle-38-arm64.yaml +++ b/ci/deps/circle-38-arm64.yaml @@ -40,7 +40,7 @@ dependencies: - odfpy>=1.4.1 - pandas-gbq>=0.15.0 - psycopg2>=2.8.6 - - pyarrow>=7.0.0 + - pyarrow - pymysql>=1.0.2 # Not provided on ARM #- pyreadstat diff --git a/scripts/validate_min_versions_in_sync.py b/scripts/validate_min_versions_in_sync.py index 4b7e8c10dc9cc..b1dc734994b69 100755 --- a/scripts/validate_min_versions_in_sync.py +++ b/scripts/validate_min_versions_in_sync.py @@ -37,6 +37,7 @@ EXCLUSION_LIST = { "python=3.8[build=*_pypy]": None, "tzdata": None, + "pyarrow": None, } # pandas package is not available # in pre-commit environment From 964e201d0ef7252f0d00fa6a703bbb6cd8d41bc3 Mon Sep 17 00:00:00 2001 From: doge Date: Tue, 21 Feb 2023 11:27:31 -0800 Subject: [PATCH 27/33] Add docstrings --- scripts/validate_min_versions_in_sync.py | 28 +++++++++++++----------- 1 file changed, 15 insertions(+), 13 deletions(-) diff --git a/scripts/validate_min_versions_in_sync.py b/scripts/validate_min_versions_in_sync.py index b1dc734994b69..6ca438500e095 100755 --- a/scripts/validate_min_versions_in_sync.py +++ b/scripts/validate_min_versions_in_sync.py @@ -14,7 +14,6 @@ """ from __future__ import annotations -import os import pathlib import sys @@ -52,6 +51,11 @@ def pin_min_versions_to_ci_deps(): + """ + Pin minimum versions to CI dependencies. + + Pip dependencies are not pinned. + """ all_yaml_files = list(YAML_PATH.iterdir()) all_yaml_files.append(ENV_PATH) toml_dependencies = {} @@ -62,17 +66,12 @@ def pin_min_versions_to_ci_deps(): yaml_file_data = yaml_f.read() yaml_file = yaml.safe_load(yaml_file_data) yaml_dependencies = yaml_file["dependencies"] - res = [] - [res.append(x) for x in yaml_dependencies if x not in res] - yaml_dependencies = res yaml_map = get_yaml_map_from(yaml_dependencies) toml_map = get_toml_map_from(toml_dependencies) - yaml_file_data = pin_min_versions_to_yaml_file( - yaml_map, toml_map, yaml_file_data - ) - os.remove(curr_file) with open(curr_file, "w") as f: - f.write(yaml_file_data) + f.write( + pin_min_versions_to_yaml_file(yaml_map, toml_map, yaml_file_data) + ) def get_toml_map_from(toml_dic) -> dict[str, str]: @@ -103,7 +102,11 @@ def get_operator_from(dependency) -> str: def get_yaml_map_from(yaml_dic) -> dict[str, str]: yaml_map = {} for dependency in yaml_dic: - if type(dependency) == dict or dependency in EXCLUSION_LIST: + if ( + isinstance(dependency, dict) + or dependency in EXCLUSION_LIST + or dependency in yaml_map + ): continue search_text = str(dependency) operator = get_operator_from(search_text) @@ -124,7 +127,7 @@ def get_yaml_map_from(yaml_dic) -> dict[str, str]: def clean_version_list(yaml_versions, toml_version): - for i in range(len(yaml_versions)): + for i, _ in enumerate(yaml_versions): yaml_version = yaml_versions[i] operator = get_operator_from(yaml_version) if "<=" in operator or ">=" in operator: @@ -151,7 +154,6 @@ def pin_min_versions_to_yaml_file(yaml_map, toml_map, yaml_file_data): old_dep += yaml_version + ", " old_dep = old_dep[:-2] if yaml_package in toml_map: - cleaned_yaml_versions = [] min_dep = toml_map[yaml_package] if yaml_versions is None: new_dep = old_dep + ">=" + min_dep @@ -159,7 +161,7 @@ def pin_min_versions_to_yaml_file(yaml_map, toml_map, yaml_file_data): continue toml_version = version.parse(min_dep) yaml_versions = clean_version_list(yaml_versions, toml_version) - [cleaned_yaml_versions.append(x) for x in yaml_versions if "-" not in x] + cleaned_yaml_versions = [x for x in yaml_versions if "-" not in x] new_dep = yaml_package for yaml_version in cleaned_yaml_versions: new_dep += yaml_version + ", " From 47179f0602cc0d58015821163ddccce2c58e41eb Mon Sep 17 00:00:00 2001 From: doge Date: Tue, 21 Feb 2023 16:05:50 -0800 Subject: [PATCH 28/33] Remove psutil --- ci/deps/actions-310.yaml | 1 - ci/deps/actions-311.yaml | 1 - ci/deps/actions-38-downstream_compat.yaml | 1 - ci/deps/actions-38-minimum_versions.yaml | 1 - ci/deps/actions-38.yaml | 1 - ci/deps/actions-39.yaml | 1 - ci/deps/circle-38-arm64.yaml | 1 - 7 files changed, 7 deletions(-) diff --git a/ci/deps/actions-310.yaml b/ci/deps/actions-310.yaml index 5905a2f6a444d..bdb2641ec5d69 100644 --- a/ci/deps/actions-310.yaml +++ b/ci/deps/actions-310.yaml @@ -12,7 +12,6 @@ dependencies: - pytest>=7.0.0 - pytest-cov - pytest-xdist>=2.2.0 - - psutil - pytest-asyncio>=0.17.0 - boto3 diff --git a/ci/deps/actions-311.yaml b/ci/deps/actions-311.yaml index 6df4f05aa351a..af1fdcafc2c28 100644 --- a/ci/deps/actions-311.yaml +++ b/ci/deps/actions-311.yaml @@ -12,7 +12,6 @@ dependencies: - pytest>=7.0.0 - pytest-cov - pytest-xdist>=2.2.0 - - psutil - pytest-asyncio>=0.17.0 - boto3 diff --git a/ci/deps/actions-38-downstream_compat.yaml b/ci/deps/actions-38-downstream_compat.yaml index 021bb6b13595e..4c0beaed0eef8 100644 --- a/ci/deps/actions-38-downstream_compat.yaml +++ b/ci/deps/actions-38-downstream_compat.yaml @@ -13,7 +13,6 @@ dependencies: - pytest>=7.0.0 - pytest-cov - pytest-xdist>=2.2.0 - - psutil - pytest-asyncio>=0.17.0 - boto3 diff --git a/ci/deps/actions-38-minimum_versions.yaml b/ci/deps/actions-38-minimum_versions.yaml index 8812a1e59cdd4..6877d7f14f66a 100644 --- a/ci/deps/actions-38-minimum_versions.yaml +++ b/ci/deps/actions-38-minimum_versions.yaml @@ -14,7 +14,6 @@ dependencies: - pytest>=7.0.0 - pytest-cov - pytest-xdist>=2.2.0 - - psutil - pytest-asyncio>=0.17.0 - boto3 diff --git a/ci/deps/actions-38.yaml b/ci/deps/actions-38.yaml index 6a815b0a4f571..0414b9372ac61 100644 --- a/ci/deps/actions-38.yaml +++ b/ci/deps/actions-38.yaml @@ -12,7 +12,6 @@ dependencies: - pytest>=7.0.0 - pytest-cov - pytest-xdist>=2.2.0 - - psutil - pytest-asyncio>=0.17.0 - boto3 diff --git a/ci/deps/actions-39.yaml b/ci/deps/actions-39.yaml index 5d92c7eff2a57..b470c492201d6 100644 --- a/ci/deps/actions-39.yaml +++ b/ci/deps/actions-39.yaml @@ -12,7 +12,6 @@ dependencies: - pytest>=7.0.0 - pytest-cov - pytest-xdist>=2.2.0 - - psutil - pytest-asyncio>=0.17.0 - boto3 diff --git a/ci/deps/circle-38-arm64.yaml b/ci/deps/circle-38-arm64.yaml index 48bca83c3c511..06ff1398b1716 100644 --- a/ci/deps/circle-38-arm64.yaml +++ b/ci/deps/circle-38-arm64.yaml @@ -12,7 +12,6 @@ dependencies: - pytest>=7.0.0 - pytest-cov - pytest-xdist>=2.2.0 - - psutil - pytest-asyncio>=0.17.0 - boto3 From 2c39a81873ff297fb3e73b4d2b11973257c17758 Mon Sep 17 00:00:00 2001 From: doge Date: Tue, 21 Feb 2023 16:10:01 -0800 Subject: [PATCH 29/33] Fix file rewrite issue --- scripts/validate_min_versions_in_sync.py | 16 +++++++++------- 1 file changed, 9 insertions(+), 7 deletions(-) diff --git a/scripts/validate_min_versions_in_sync.py b/scripts/validate_min_versions_in_sync.py index 6ca438500e095..999550c092bfc 100755 --- a/scripts/validate_min_versions_in_sync.py +++ b/scripts/validate_min_versions_in_sync.py @@ -63,15 +63,17 @@ def pin_min_versions_to_ci_deps(): toml_dependencies = tomllib.load(toml_f) for curr_file in all_yaml_files: with open(curr_file) as yaml_f: - yaml_file_data = yaml_f.read() - yaml_file = yaml.safe_load(yaml_file_data) + yaml_start_data = yaml_f.read() + yaml_file = yaml.safe_load(yaml_start_data) yaml_dependencies = yaml_file["dependencies"] yaml_map = get_yaml_map_from(yaml_dependencies) toml_map = get_toml_map_from(toml_dependencies) - with open(curr_file, "w") as f: - f.write( - pin_min_versions_to_yaml_file(yaml_map, toml_map, yaml_file_data) - ) + yaml_result_data = pin_min_versions_to_yaml_file( + yaml_map, toml_map, yaml_start_data + ) + if yaml_result_data not in yaml_start_data: + with open(curr_file, "w") as f: + f.write(yaml_result_data) def get_toml_map_from(toml_dic) -> dict[str, str]: @@ -127,7 +129,7 @@ def get_yaml_map_from(yaml_dic) -> dict[str, str]: def clean_version_list(yaml_versions, toml_version): - for i, _ in enumerate(yaml_versions): + for i in range(len(yaml_versions)): yaml_version = yaml_versions[i] operator = get_operator_from(yaml_version) if "<=" in operator or ">=" in operator: From 554ed85e9ee9c6e06b4166978dff8859862fe281 Mon Sep 17 00:00:00 2001 From: doge Date: Tue, 21 Feb 2023 16:10:47 -0800 Subject: [PATCH 30/33] Refactor test data --- .../deps_expected_duplicate_package.yaml} | 0 .../deps_expected_no_version.yaml} | 0 .../deps_expected_random.yaml} | 0 .../deps_expected_range.yaml} | 0 .../deps_expected_same_version.yaml} | 0 .../deps_minimum.toml} | 0 .../deps_unmodified_duplicate_package.yaml} | 0 .../deps_unmodified_no_version.yaml} | 0 .../deps_unmodified_random.yaml} | 0 .../deps_unmodified_range.yaml} | 0 .../deps_unmodified_same_version.yaml} | 0 .../test_validate_min_versions_in_sync.py | 60 +++++++------------ 12 files changed, 20 insertions(+), 40 deletions(-) rename scripts/tests/{dummy_test_files/dummy_yaml_expected_file_3.yaml => data/deps_expected_duplicate_package.yaml} (100%) rename scripts/tests/{dummy_test_files/dummy_yaml_expected_file_4.yaml => data/deps_expected_no_version.yaml} (100%) rename scripts/tests/{dummy_test_files/dummy_yaml_expected_file_1.yaml => data/deps_expected_random.yaml} (100%) rename scripts/tests/{dummy_test_files/dummy_yaml_expected_file_5.yaml => data/deps_expected_range.yaml} (100%) rename scripts/tests/{dummy_test_files/dummy_yaml_expected_file_2.yaml => data/deps_expected_same_version.yaml} (100%) rename scripts/tests/{dummy_test_files/dummy_toml_file.toml => data/deps_minimum.toml} (100%) rename scripts/tests/{dummy_test_files/dummy_yaml_unmodified_file_3.yaml => data/deps_unmodified_duplicate_package.yaml} (100%) rename scripts/tests/{dummy_test_files/dummy_yaml_unmodified_file_4.yaml => data/deps_unmodified_no_version.yaml} (100%) rename scripts/tests/{dummy_test_files/dummy_yaml_unmodified_file_1.yaml => data/deps_unmodified_random.yaml} (100%) rename scripts/tests/{dummy_test_files/dummy_yaml_unmodified_file_5.yaml => data/deps_unmodified_range.yaml} (100%) rename scripts/tests/{dummy_test_files/dummy_yaml_unmodified_file_2.yaml => data/deps_unmodified_same_version.yaml} (100%) diff --git a/scripts/tests/dummy_test_files/dummy_yaml_expected_file_3.yaml b/scripts/tests/data/deps_expected_duplicate_package.yaml similarity index 100% rename from scripts/tests/dummy_test_files/dummy_yaml_expected_file_3.yaml rename to scripts/tests/data/deps_expected_duplicate_package.yaml diff --git a/scripts/tests/dummy_test_files/dummy_yaml_expected_file_4.yaml b/scripts/tests/data/deps_expected_no_version.yaml similarity index 100% rename from scripts/tests/dummy_test_files/dummy_yaml_expected_file_4.yaml rename to scripts/tests/data/deps_expected_no_version.yaml diff --git a/scripts/tests/dummy_test_files/dummy_yaml_expected_file_1.yaml b/scripts/tests/data/deps_expected_random.yaml similarity index 100% rename from scripts/tests/dummy_test_files/dummy_yaml_expected_file_1.yaml rename to scripts/tests/data/deps_expected_random.yaml diff --git a/scripts/tests/dummy_test_files/dummy_yaml_expected_file_5.yaml b/scripts/tests/data/deps_expected_range.yaml similarity index 100% rename from scripts/tests/dummy_test_files/dummy_yaml_expected_file_5.yaml rename to scripts/tests/data/deps_expected_range.yaml diff --git a/scripts/tests/dummy_test_files/dummy_yaml_expected_file_2.yaml b/scripts/tests/data/deps_expected_same_version.yaml similarity index 100% rename from scripts/tests/dummy_test_files/dummy_yaml_expected_file_2.yaml rename to scripts/tests/data/deps_expected_same_version.yaml diff --git a/scripts/tests/dummy_test_files/dummy_toml_file.toml b/scripts/tests/data/deps_minimum.toml similarity index 100% rename from scripts/tests/dummy_test_files/dummy_toml_file.toml rename to scripts/tests/data/deps_minimum.toml diff --git a/scripts/tests/dummy_test_files/dummy_yaml_unmodified_file_3.yaml b/scripts/tests/data/deps_unmodified_duplicate_package.yaml similarity index 100% rename from scripts/tests/dummy_test_files/dummy_yaml_unmodified_file_3.yaml rename to scripts/tests/data/deps_unmodified_duplicate_package.yaml diff --git a/scripts/tests/dummy_test_files/dummy_yaml_unmodified_file_4.yaml b/scripts/tests/data/deps_unmodified_no_version.yaml similarity index 100% rename from scripts/tests/dummy_test_files/dummy_yaml_unmodified_file_4.yaml rename to scripts/tests/data/deps_unmodified_no_version.yaml diff --git a/scripts/tests/dummy_test_files/dummy_yaml_unmodified_file_1.yaml b/scripts/tests/data/deps_unmodified_random.yaml similarity index 100% rename from scripts/tests/dummy_test_files/dummy_yaml_unmodified_file_1.yaml rename to scripts/tests/data/deps_unmodified_random.yaml diff --git a/scripts/tests/dummy_test_files/dummy_yaml_unmodified_file_5.yaml b/scripts/tests/data/deps_unmodified_range.yaml similarity index 100% rename from scripts/tests/dummy_test_files/dummy_yaml_unmodified_file_5.yaml rename to scripts/tests/data/deps_unmodified_range.yaml diff --git a/scripts/tests/dummy_test_files/dummy_yaml_unmodified_file_2.yaml b/scripts/tests/data/deps_unmodified_same_version.yaml similarity index 100% rename from scripts/tests/dummy_test_files/dummy_yaml_unmodified_file_2.yaml rename to scripts/tests/data/deps_unmodified_same_version.yaml diff --git a/scripts/tests/test_validate_min_versions_in_sync.py b/scripts/tests/test_validate_min_versions_in_sync.py index ae1613b967b8c..3e0e998a823f5 100644 --- a/scripts/tests/test_validate_min_versions_in_sync.py +++ b/scripts/tests/test_validate_min_versions_in_sync.py @@ -19,50 +19,30 @@ @pytest.mark.parametrize( "src_toml, src_yaml, expected_yaml", [ - ( # random - pathlib.Path("scripts/tests/dummy_test_files/dummy_toml_file.toml"), - pathlib.Path( - "scripts/tests/dummy_test_files/dummy_yaml_unmodified_file_1.yaml" - ), - pathlib.Path( - "scripts/tests/dummy_test_files/dummy_yaml_expected_file_1.yaml" - ), + ( + pathlib.Path("scripts/tests/data/deps_minimum.toml"), + pathlib.Path("scripts/tests/data/deps_unmodified_random.yaml"), + pathlib.Path("scripts/tests/data/deps_expected_random.yaml"), ), - ( # same version - pathlib.Path("scripts/tests/dummy_test_files/dummy_toml_file.toml"), - pathlib.Path( - "scripts/tests/dummy_test_files/dummy_yaml_unmodified_file_2.yaml" - ), - pathlib.Path( - "scripts/tests/dummy_test_files/dummy_yaml_expected_file_2.yaml" - ), + ( + pathlib.Path("scripts/tests/data/deps_minimum.toml"), + pathlib.Path("scripts/tests/data/deps_unmodified_same_version.yaml"), + pathlib.Path("scripts/tests/data/deps_expected_same_version.yaml"), ), - ( # duplicate package - pathlib.Path("scripts/tests/dummy_test_files/dummy_toml_file.toml"), - pathlib.Path( - "scripts/tests/dummy_test_files/dummy_yaml_unmodified_file_3.yaml" - ), - pathlib.Path( - "scripts/tests/dummy_test_files/dummy_yaml_expected_file_3.yaml" - ), + ( + pathlib.Path("scripts/tests/data/deps_minimum.toml"), + pathlib.Path("scripts/tests/data/deps_unmodified_duplicate_package.yaml"), + pathlib.Path("scripts/tests/data/deps_expected_duplicate_package.yaml"), ), - ( # empty version - pathlib.Path("scripts/tests/dummy_test_files/dummy_toml_file.toml"), - pathlib.Path( - "scripts/tests/dummy_test_files/dummy_yaml_unmodified_file_4.yaml" - ), - pathlib.Path( - "scripts/tests/dummy_test_files/dummy_yaml_expected_file_4.yaml" - ), + ( + pathlib.Path("scripts/tests/data/deps_minimum.toml"), + pathlib.Path("scripts/tests/data/deps_unmodified_no_version.yaml"), + pathlib.Path("scripts/tests/data/deps_expected_no_version.yaml"), ), - ( # range - pathlib.Path("scripts/tests/dummy_test_files/dummy_toml_file.toml"), - pathlib.Path( - "scripts/tests/dummy_test_files/dummy_yaml_unmodified_file_5.yaml" - ), - pathlib.Path( - "scripts/tests/dummy_test_files/dummy_yaml_expected_file_5.yaml" - ), + ( + pathlib.Path("scripts/tests/data/deps_minimum.toml"), + pathlib.Path("scripts/tests/data/deps_unmodified_range.yaml"), + pathlib.Path("scripts/tests/data/deps_expected_range.yaml"), ), ], ) From 985730fde3b4a60a9ce820851fbecbc47b4b0dff Mon Sep 17 00:00:00 2001 From: MarcoGorelli <> Date: Thu, 23 Feb 2023 10:31:50 +0000 Subject: [PATCH 31/33] minor fixups --- .pre-commit-config.yaml | 5 +- ci/deps/actions-310.yaml | 4 +- ci/deps/actions-311.yaml | 4 +- ci/deps/actions-38-downstream_compat.yaml | 4 +- ci/deps/actions-38.yaml | 2 +- ci/deps/actions-39.yaml | 4 +- ci/deps/circle-38-arm64.yaml | 4 +- environment.yml | 1 + pyproject.toml | 1 + requirements-dev.txt | 3 +- scripts/generate_pip_deps_from_conda.py | 9 +- .../test_validate_min_versions_in_sync.py | 12 +-- scripts/validate_min_versions_in_sync.py | 96 +++++++++++-------- 13 files changed, 87 insertions(+), 62 deletions(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index a49af46efcb4b..b1028ea9f52c3 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -313,7 +313,7 @@ repos: entry: python scripts/generate_pip_deps_from_conda.py files: ^(environment.yml|requirements-dev.txt)$ pass_filenames: false - additional_dependencies: [pyyaml, toml] + additional_dependencies: [tomli, pyyaml] - id: title-capitalization name: Validate correct capitalization among titles in documentation entry: python scripts/validate_rst_title_capitalization.py @@ -391,10 +391,11 @@ repos: types: [yaml] - id: validate-min-versions-in-sync name: Check minimum version of dependencies are aligned - entry: python scripts/validate_min_versions_in_sync.py + entry: python -m scripts.validate_min_versions_in_sync language: python files: ^(ci/deps/actions-.*-minimum_versions\.yaml|pandas/compat/_optional\.py)$ additional_dependencies: [tomli, pyyaml] + pass_filenames: false - id: validate-errors-locations name: Validate errors locations description: Validate errors are in appropriate locations. diff --git a/ci/deps/actions-310.yaml b/ci/deps/actions-310.yaml index bdb2641ec5d69..64f9a3fd1ffbc 100644 --- a/ci/deps/actions-310.yaml +++ b/ci/deps/actions-310.yaml @@ -35,7 +35,7 @@ dependencies: - matplotlib>=3.6.1, <3.7.0 - numba>=0.53.1 - numexpr>=2.7.3 - - openpyxl>=3.0.7, <3.1.1 + - openpyxl<3.1.1, >=3.0.7 - odfpy>=1.4.1 - pandas-gbq>=0.15.0 - psycopg2>=2.8.6 @@ -47,7 +47,7 @@ dependencies: - pyxlsb>=1.0.8 - s3fs>=2021.08.0 - scipy>=1.7.1 - - sqlalchemy<1.4.46, >=1.4.16 + - sqlalchemy>=1.4.16 - tabulate>=0.8.9 - tzdata>=2022a - xarray>=0.21.0 diff --git a/ci/deps/actions-311.yaml b/ci/deps/actions-311.yaml index af1fdcafc2c28..d474df1e75655 100644 --- a/ci/deps/actions-311.yaml +++ b/ci/deps/actions-311.yaml @@ -35,7 +35,7 @@ dependencies: - matplotlib>=3.6.1, <3.7.0 # - numba not compatible with 3.11 - numexpr>=2.7.3 - - openpyxl>=3.0.7, <3.1.1 + - openpyxl<3.1.1, >=3.0.7 - odfpy>=1.4.1 - pandas-gbq>=0.15.0 - psycopg2>=2.8.6 @@ -47,7 +47,7 @@ dependencies: - pyxlsb>=1.0.8 - s3fs>=2021.08.0 - scipy>=1.7.1 - - sqlalchemy<1.4.46, >=1.4.16 + - sqlalchemy>=1.4.16 - tabulate>=0.8.9 - tzdata>=2022a - xarray>=0.21.0 diff --git a/ci/deps/actions-38-downstream_compat.yaml b/ci/deps/actions-38-downstream_compat.yaml index 4c0beaed0eef8..a9cd4c93dd604 100644 --- a/ci/deps/actions-38-downstream_compat.yaml +++ b/ci/deps/actions-38-downstream_compat.yaml @@ -33,10 +33,10 @@ dependencies: - gcsfs>=2021.07.0 - jinja2>=3.0.0 - lxml>=4.6.3 - - matplotlib>=3.6.1 + - matplotlib>=3.6.1, <3.7.0 - numba>=0.53.1 - numexpr>=2.7.3 - - openpyxl>=3.0.7, <3.1.1 + - openpyxl<3.1.1, >=3.0.7 - odfpy>=1.4.1 - psycopg2>=2.8.6 - pyarrow diff --git a/ci/deps/actions-38.yaml b/ci/deps/actions-38.yaml index 0414b9372ac61..ccde0f57f7bc4 100644 --- a/ci/deps/actions-38.yaml +++ b/ci/deps/actions-38.yaml @@ -35,7 +35,7 @@ dependencies: - matplotlib>=3.6.1, <3.7.0 - numba>=0.53.1 - numexpr>=2.7.3 - - openpyxl>=3.0.7, <3.1.1 + - openpyxl<3.1.1, >=3.0.7 - odfpy>=1.4.1 - pandas-gbq>=0.15.0 - psycopg2>=2.8.6 diff --git a/ci/deps/actions-39.yaml b/ci/deps/actions-39.yaml index b470c492201d6..aeb887d7ec1ab 100644 --- a/ci/deps/actions-39.yaml +++ b/ci/deps/actions-39.yaml @@ -35,7 +35,7 @@ dependencies: - matplotlib>=3.6.1, <3.7.0 - numba>=0.53.1 - numexpr>=2.7.3 - - openpyxl>=3.0.7, <3.1.1 + - openpyxl<3.1.1, >=3.0.7 - odfpy>=1.4.1 - pandas-gbq>=0.15.0 - psycopg2>=2.8.6 @@ -47,7 +47,7 @@ dependencies: - pyxlsb>=1.0.8 - s3fs>=2021.08.0 - scipy>=1.7.1 - - sqlalchemy<1.4.46, >=1.4.16 + - sqlalchemy>=1.4.16 - tabulate>=0.8.9 - tzdata>=2022a - xarray>=0.21.0 diff --git a/ci/deps/circle-38-arm64.yaml b/ci/deps/circle-38-arm64.yaml index 06ff1398b1716..0d1a5f765b5ce 100644 --- a/ci/deps/circle-38-arm64.yaml +++ b/ci/deps/circle-38-arm64.yaml @@ -35,7 +35,7 @@ dependencies: - matplotlib>=3.6.1, <3.7.0 - numba>=0.53.1 - numexpr>=2.7.3 - - openpyxl>=3.0.7, <3.1.1 + - openpyxl<3.1.1, >=3.0.7 - odfpy>=1.4.1 - pandas-gbq>=0.15.0 - psycopg2>=2.8.6 @@ -48,7 +48,7 @@ dependencies: - pyxlsb>=1.0.8 - s3fs>=2021.08.0 - scipy>=1.7.1 - - sqlalchemy<1.4.46, >=1.4.16 + - sqlalchemy>=1.4.16 - tabulate>=0.8.9 - xarray>=0.21.0 - xlrd>=2.0.1 diff --git a/environment.yml b/environment.yml index 64971a86fe8df..384f3f5d7031f 100644 --- a/environment.yml +++ b/environment.yml @@ -97,6 +97,7 @@ dependencies: - types-python-dateutil - types-PyMySQL - types-pytz + - types-PyYAML - types-setuptools # documentation (jupyter notebooks) diff --git a/pyproject.toml b/pyproject.toml index 9d4166b033128..7d0b917c2ccf5 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -287,6 +287,7 @@ disable = [ "broad-except", "c-extension-no-member", "comparison-with-itself", + "consider-using-enumerate", "import-error", "import-outside-toplevel", "invalid-name", diff --git a/requirements-dev.txt b/requirements-dev.txt index c52abae23b84b..ca3c4d36bfd98 100644 --- a/requirements-dev.txt +++ b/requirements-dev.txt @@ -40,7 +40,7 @@ python-snappy>=0.6.0 pyxlsb>=1.0.8 s3fs>=2021.08.0 scipy>=1.7.1 -sqlalchemy>=1.4.16 +SQLAlchemy>=1.4.16 tabulate>=0.8.9 tzdata>=2022.1 xarray>=0.21.0 @@ -72,6 +72,7 @@ sphinx-copybutton types-python-dateutil types-PyMySQL types-pytz +types-PyYAML types-setuptools nbconvert>=6.4.5 nbsphinx diff --git a/scripts/generate_pip_deps_from_conda.py b/scripts/generate_pip_deps_from_conda.py index 8190104428724..2ca4455158db5 100755 --- a/scripts/generate_pip_deps_from_conda.py +++ b/scripts/generate_pip_deps_from_conda.py @@ -17,7 +17,10 @@ import re import sys -import toml +if sys.version_info >= (3, 11): + import tomllib +else: + import tomli as tomllib import yaml EXCLUDE = {"python", "c-compiler", "cxx-compiler"} @@ -27,6 +30,7 @@ "psycopg2": "psycopg2-binary", "dask-core": "dask", "seaborn-base": "seaborn", + "sqlalchemy": "SQLAlchemy", } @@ -105,7 +109,8 @@ def generate_pip_from_conda( pip_content = header + "\n".join(pip_deps) + "\n" # add setuptools to requirements-dev.txt - meta = toml.load(pathlib.Path(conda_path.parent, "pyproject.toml")) + with open(pathlib.Path(conda_path.parent, "pyproject.toml"), "rb") as fd: + meta = tomllib.load(fd) for requirement in meta["build-system"]["requires"]: if "setuptools" in requirement: pip_content += requirement diff --git a/scripts/tests/test_validate_min_versions_in_sync.py b/scripts/tests/test_validate_min_versions_in_sync.py index 3e0e998a823f5..13e8965bb7591 100644 --- a/scripts/tests/test_validate_min_versions_in_sync.py +++ b/scripts/tests/test_validate_min_versions_in_sync.py @@ -51,13 +51,11 @@ def test_pin_min_versions_to_yaml_file(src_toml, src_yaml, expected_yaml): toml_map = tomllib.load(toml_f) with open(src_yaml) as yaml_f: yaml_file_data = yaml_f.read() - yaml_file = yaml.safe_load(yaml_file_data) - yaml_dependencies = yaml_file["dependencies"] - yaml_map = get_yaml_map_from(yaml_dependencies) - toml_map = get_toml_map_from(toml_map) - result_yaml_file = pin_min_versions_to_yaml_file( - yaml_map, toml_map, yaml_file_data - ) + yaml_file = yaml.safe_load(yaml_file_data) + yaml_dependencies = yaml_file["dependencies"] + yaml_map = get_yaml_map_from(yaml_dependencies) + toml_map = get_toml_map_from(toml_map) + result_yaml_file = pin_min_versions_to_yaml_file(yaml_map, toml_map, yaml_file_data) with open(expected_yaml) as yaml_f: dummy_yaml_expected_file_1 = yaml_f.read() assert result_yaml_file == dummy_yaml_expected_file_1 diff --git a/scripts/validate_min_versions_in_sync.py b/scripts/validate_min_versions_in_sync.py index 999550c092bfc..a62017ae81cce 100755 --- a/scripts/validate_min_versions_in_sync.py +++ b/scripts/validate_min_versions_in_sync.py @@ -24,6 +24,10 @@ else: import tomli as tomllib +from typing import Any + +from scripts.generate_pip_deps_from_conda import RENAME + DOC_PATH = pathlib.Path("doc/source/getting_started/install.rst").resolve() CI_PATH = next( pathlib.Path("ci/deps").absolute().glob("actions-*-minimum_versions.yaml") @@ -50,7 +54,7 @@ import _optional -def pin_min_versions_to_ci_deps(): +def pin_min_versions_to_ci_deps() -> int: """ Pin minimum versions to CI dependencies. @@ -61,22 +65,25 @@ def pin_min_versions_to_ci_deps(): toml_dependencies = {} with open(SETUP_PATH, "rb") as toml_f: toml_dependencies = tomllib.load(toml_f) + ret = 0 for curr_file in all_yaml_files: with open(curr_file) as yaml_f: yaml_start_data = yaml_f.read() - yaml_file = yaml.safe_load(yaml_start_data) - yaml_dependencies = yaml_file["dependencies"] - yaml_map = get_yaml_map_from(yaml_dependencies) - toml_map = get_toml_map_from(toml_dependencies) - yaml_result_data = pin_min_versions_to_yaml_file( - yaml_map, toml_map, yaml_start_data - ) - if yaml_result_data not in yaml_start_data: - with open(curr_file, "w") as f: - f.write(yaml_result_data) + yaml_file = yaml.safe_load(yaml_start_data) + yaml_dependencies = yaml_file["dependencies"] + yaml_map = get_yaml_map_from(yaml_dependencies) + toml_map = get_toml_map_from(toml_dependencies) + yaml_result_data = pin_min_versions_to_yaml_file( + yaml_map, toml_map, yaml_start_data + ) + if yaml_result_data != yaml_start_data: + with open(curr_file, "w") as f: + f.write(yaml_result_data) + ret |= 1 + return ret -def get_toml_map_from(toml_dic) -> dict[str, str]: +def get_toml_map_from(toml_dic: dict[str, Any]) -> dict[str, str]: toml_deps = {} toml_dependencies = set(toml_dic["project"]["optional-dependencies"]["all"]) for dependency in toml_dependencies: @@ -85,7 +92,7 @@ def get_toml_map_from(toml_dic) -> dict[str, str]: return toml_deps -def get_operator_from(dependency) -> str: +def get_operator_from(dependency: str) -> str | None: if "<=" in dependency: operator = "<=" elif ">=" in dependency: @@ -101,8 +108,8 @@ def get_operator_from(dependency) -> str: return operator -def get_yaml_map_from(yaml_dic) -> dict[str, str]: - yaml_map = {} +def get_yaml_map_from(yaml_dic: list[str]) -> dict[str, list[str] | None]: + yaml_map: dict[str, list[str] | None] = {} for dependency in yaml_dic: if ( isinstance(dependency, dict) @@ -115,6 +122,7 @@ def get_yaml_map_from(yaml_dic) -> dict[str, str]: if "," in dependency: yaml_dependency, yaml_version1 = search_text.split(",") operator = get_operator_from(yaml_dependency) + assert operator is not None yaml_package, yaml_version2 = yaml_dependency.split(operator) yaml_version2 = operator + yaml_version2 yaml_map[yaml_package] = [yaml_version1, yaml_version2] @@ -128,10 +136,13 @@ def get_yaml_map_from(yaml_dic) -> dict[str, str]: return yaml_map -def clean_version_list(yaml_versions, toml_version): +def clean_version_list( + yaml_versions: list[str], toml_version: version.Version +) -> list[str]: for i in range(len(yaml_versions)): yaml_version = yaml_versions[i] operator = get_operator_from(yaml_version) + assert operator is not None if "<=" in operator or ">=" in operator: yaml_version = yaml_version[2:] else: @@ -145,7 +156,9 @@ def clean_version_list(yaml_versions, toml_version): return yaml_versions -def pin_min_versions_to_yaml_file(yaml_map, toml_map, yaml_file_data): +def pin_min_versions_to_yaml_file( + yaml_map: dict[str, list[str] | None], toml_map: dict[str, str], yaml_file_data: str +) -> str: data = yaml_file_data for yaml_package, yaml_versions in yaml_map.items(): if yaml_package in EXCLUSION_LIST: @@ -155,24 +168,28 @@ def pin_min_versions_to_yaml_file(yaml_map, toml_map, yaml_file_data): for yaml_version in yaml_versions: old_dep += yaml_version + ", " old_dep = old_dep[:-2] - if yaml_package in toml_map: + if RENAME.get(yaml_package, yaml_package) in toml_map: + min_dep = toml_map[RENAME.get(yaml_package, yaml_package)] + elif yaml_package in toml_map: min_dep = toml_map[yaml_package] - if yaml_versions is None: - new_dep = old_dep + ">=" + min_dep - data = data.replace(old_dep, new_dep, 1) - continue - toml_version = version.parse(min_dep) - yaml_versions = clean_version_list(yaml_versions, toml_version) - cleaned_yaml_versions = [x for x in yaml_versions if "-" not in x] - new_dep = yaml_package - for yaml_version in cleaned_yaml_versions: - new_dep += yaml_version + ", " - operator = get_operator_from(new_dep) - if operator != "=": - new_dep += ">=" + min_dep - else: - new_dep = new_dep[:-2] - data = data.replace(old_dep, new_dep) + else: + continue + if yaml_versions is None: + new_dep = old_dep + ">=" + min_dep + data = data.replace(old_dep, new_dep, 1) + continue + toml_version = version.parse(min_dep) + yaml_versions = clean_version_list(yaml_versions, toml_version) + cleaned_yaml_versions = [x for x in yaml_versions if "-" not in x] + new_dep = yaml_package + for yaml_version in cleaned_yaml_versions: + new_dep += yaml_version + ", " + operator = get_operator_from(new_dep) + if operator != "=": + new_dep += ">=" + min_dep + else: + new_dep = new_dep[:-2] + data = data.replace(old_dep, new_dep) return data @@ -245,8 +262,9 @@ def get_versions_from_toml() -> dict[str, str]: return optional_dependencies -def main(): - pin_min_versions_to_ci_deps() +def main() -> int: + ret = 0 + ret |= pin_min_versions_to_ci_deps() with open(CI_PATH, encoding="utf-8") as f: _, ci_optional = get_versions_from_ci(f.readlines()) code_optional = get_versions_from_code() @@ -272,9 +290,9 @@ def main(): f"{CODE_PATH}: {code_optional.get(package, 'Not specified')}\n" f"{SETUP_PATH}: {setup_optional.get(package, 'Not specified')}\n\n" ) - sys.exit(1) - sys.exit(0) + ret |= 1 + return ret if __name__ == "__main__": - main() + sys.exit(main()) From 07c68b31a830af55076620740c2e4a930f4aa526 Mon Sep 17 00:00:00 2001 From: MarcoGorelli <> Date: Thu, 23 Feb 2023 11:15:37 +0000 Subject: [PATCH 32/33] fixup --- environment.yml | 7 +++---- requirements-dev.txt | 7 +++---- 2 files changed, 6 insertions(+), 8 deletions(-) diff --git a/environment.yml b/environment.yml index 384f3f5d7031f..e748d20d6d6f0 100644 --- a/environment.yml +++ b/environment.yml @@ -14,7 +14,6 @@ dependencies: - pytest>=7.0.0 - pytest-cov - pytest-xdist>=2.2.0 - - psutil - pytest-asyncio>=0.17.0 - coverage @@ -39,11 +38,11 @@ dependencies: - matplotlib>=3.6.1, <3.7.0 - numba>=0.53.1 - numexpr>=2.7.3 # pin for "Run checks on imported code" job - - openpyxl>=3.0.7, <3.1.1 + - openpyxl<3.1.1, >=3.0.7 - odfpy>=1.4.1 - py - psycopg2>=2.8.6 - - pyarrow>=7.0.0 + - pyarrow - pymysql>=1.0.2 - pyreadstat>=1.1.2 - pytables>=3.6.1 @@ -110,7 +109,7 @@ dependencies: - ipykernel # web - - jinja2>=3.0.0 # in optional dependencies, but documented here as needed + - jinja2 # in optional dependencies, but documented here as needed - markdown - feedparser - pyyaml diff --git a/requirements-dev.txt b/requirements-dev.txt index ca3c4d36bfd98..0329588de17fd 100644 --- a/requirements-dev.txt +++ b/requirements-dev.txt @@ -7,7 +7,6 @@ cython==0.29.32 pytest>=7.0.0 pytest-cov pytest-xdist>=2.2.0 -psutil pytest-asyncio>=0.17.0 coverage python-dateutil @@ -28,11 +27,11 @@ lxml>=4.6.3 matplotlib>=3.6.1, <3.7.0 numba>=0.53.1 numexpr>=2.7.3 -openpyxl>=3.0.7, <3.1.1 +openpyxl<3.1.1, >=3.0.7 odfpy>=1.4.1 py psycopg2-binary>=2.8.6 -pyarrow>=7.0.0 +pyarrow pymysql>=1.0.2 pyreadstat>=1.1.2 tables>=3.6.1 @@ -81,7 +80,7 @@ ipywidgets nbformat notebook>=6.0.3 ipykernel -jinja2>=3.0.0 +jinja2 markdown feedparser pyyaml From 74965c23aafb1ef909e7e54cbef2c0a83fdbf24f Mon Sep 17 00:00:00 2001 From: MarcoGorelli <> Date: Thu, 23 Feb 2023 11:19:48 +0000 Subject: [PATCH 33/33] improve typing --- scripts/validate_min_versions_in_sync.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/scripts/validate_min_versions_in_sync.py b/scripts/validate_min_versions_in_sync.py index a62017ae81cce..3c12f17fe72cf 100755 --- a/scripts/validate_min_versions_in_sync.py +++ b/scripts/validate_min_versions_in_sync.py @@ -108,7 +108,9 @@ def get_operator_from(dependency: str) -> str | None: return operator -def get_yaml_map_from(yaml_dic: list[str]) -> dict[str, list[str] | None]: +def get_yaml_map_from( + yaml_dic: list[str | dict[str, list[str]]] +) -> dict[str, list[str] | None]: yaml_map: dict[str, list[str] | None] = {} for dependency in yaml_dic: if (