From b8bd2bdf0130d0e9eaad94cf075c0ab2d3a8083c Mon Sep 17 00:00:00 2001 From: Gary Miguel Date: Fri, 16 Jun 2023 21:19:54 +0000 Subject: [PATCH 1/2] NOT WORKING: use bazel to install orjson If we can figure out how to get this to work at all, should probably also use a pythoon hermetic toolchain. --- WORKSPACE | 17 ++++++++++-- python_requirements_lock.bzl | 52 ++++++++++++++++++++++++++++++++++++ python_requirements_lock.txt | 1 + refresh_compile_commands.bzl | 4 +-- workspace_setup.bzl | 34 ++++++++++++++++++++--- 5 files changed, 101 insertions(+), 7 deletions(-) create mode 100755 python_requirements_lock.bzl create mode 100644 python_requirements_lock.txt diff --git a/WORKSPACE b/WORKSPACE index 0cc57de..49c9a00 100644 --- a/WORKSPACE +++ b/WORKSPACE @@ -1,8 +1,21 @@ # This file existed originally to enable quick local development via local_repository. - # See ./ImplementationReadme.md for details on local development. - # Why? local_repository didn't work without a WORKSPACE, and new_local_repository required overwriting the BUILD file (as of Bazel 5.0). +# See ./ImplementationReadme.md for details on local development. +# Why? local_repository didn't work without a WORKSPACE, and new_local_repository required overwriting the BUILD file (as of Bazel 5.0). workspace(name = "hedron_compile_commands") load("@hedron_compile_commands//:workspace_setup.bzl", "hedron_compile_commands_setup") + hedron_compile_commands_setup() + +# For re-generating python_requirements_lock.bzl: +# * update python_requirements_lock.txt +# * Un-comment the below +# * run `bazel build @pip//...`, +# * cp external/pip/requirements.bzl python_requirements_lock.bzl + +# load("@rules_python//python:pip.bzl", "pip_parse") +# pip_parse( +# name = "pip", +# requirements_lock = "//:python_requirements_lock.txt", +# ) diff --git a/python_requirements_lock.bzl b/python_requirements_lock.bzl new file mode 100755 index 0000000..8adb5a7 --- /dev/null +++ b/python_requirements_lock.bzl @@ -0,0 +1,52 @@ +"""Starlark representation of locked requirements. + +@generated by rules_python pip_parse repository rule +from @//:python_requirements_lock.txt +""" + +load("@rules_python//python/pip_install:pip_repository.bzl", "whl_library") + +all_requirements = ["@pip_orjson//:pkg"] + +all_whl_requirements = ["@pip_orjson//:whl"] + +_packages = [("pip_orjson", "orjson==3.9.1")] +_config = {"download_only": False, "enable_implicit_namespace_pkgs": False, "environment": {}, "extra_pip_args": [], "isolated": True, "pip_data_exclude": [], "python_interpreter": "python3", "quiet": True, "repo": "pip", "repo_prefix": "pip_", "timeout": 600} +_annotations = {} + +def _clean_name(name): + return name.replace("-", "_").replace(".", "_").lower() + +def requirement(name): + return "@pip_" + _clean_name(name) + "//:pkg" + +def whl_requirement(name): + return "@pip_" + _clean_name(name) + "//:whl" + +def data_requirement(name): + return "@pip_" + _clean_name(name) + "//:data" + +def dist_info_requirement(name): + return "@pip_" + _clean_name(name) + "//:dist_info" + +def entry_point(pkg, script = None): + if not script: + script = pkg + return "@pip_" + _clean_name(pkg) + "//:rules_python_wheel_entry_point_" + script + +def _get_annotation(requirement): + # This expects to parse `setuptools==58.2.0 --hash=sha256:2551203ae6955b9876741a26ab3e767bb3242dafe86a32a749ea0d78b6792f11` + # down to `setuptools`. + name = requirement.split(" ")[0].split("=")[0].split("[")[0] + return _annotations.get(name) + +def install_deps(**whl_library_kwargs): + whl_config = dict(_config) + whl_config.update(whl_library_kwargs) + for name, requirement in _packages: + whl_library( + name = name, + requirement = requirement, + annotation = _get_annotation(requirement), + **whl_config + ) diff --git a/python_requirements_lock.txt b/python_requirements_lock.txt new file mode 100644 index 0000000..c86bdcb --- /dev/null +++ b/python_requirements_lock.txt @@ -0,0 +1 @@ +orjson==3.9.1 diff --git a/refresh_compile_commands.bzl b/refresh_compile_commands.bzl index b9c5d32..10b88a3 100644 --- a/refresh_compile_commands.bzl +++ b/refresh_compile_commands.bzl @@ -52,11 +52,11 @@ refresh_compile_commands( ``` """ - ######################################## # Implementation load("@bazel_tools//tools/cpp:toolchain_utils.bzl", "find_cpp_toolchain") +load("@rules_python//python:defs.bzl", "py_binary") def refresh_compile_commands( name, @@ -80,7 +80,7 @@ def refresh_compile_commands( # Generate runnable python script from template script_name = name + ".py" _expand_template(name = script_name, labels_to_flags = targets, exclude_headers = exclude_headers, exclude_external_sources = exclude_external_sources, **kwargs) - native.py_binary(name = name, srcs = [script_name], **kwargs) + py_binary(name = name, srcs = [script_name], deps = ["@pip_orjson//:pkg"], **kwargs) def _expand_template_impl(ctx): """Inject targets of interest into refresh.template.py, and set it up to be run.""" diff --git a/workspace_setup.bzl b/workspace_setup.bzl index bcce1d0..cd09c1c 100644 --- a/workspace_setup.bzl +++ b/workspace_setup.bzl @@ -1,5 +1,16 @@ +"""Workspace setup macros.""" + # Do not change the filename; it is part of the user interface. +load("@bazel_tools//tools/build_defs/repo:http.bzl", _http_archive = "http_archive") +load("@bazel_tools//tools/build_defs/repo:utils.bzl", "maybe") + +# TODO: This fails to load because it relies on rules_python already being defined. +load("//:python_requirements_lock.bzl", install_python_deps = "install_deps") + +def http_archive(**kwargs): + maybe(_http_archive, **kwargs) + def hedron_compile_commands_setup(): """Set up a WORKSPACE to have hedron_compile_commands used within it.""" @@ -8,6 +19,23 @@ def hedron_compile_commands_setup(): # README.md (for users) # WORKSPACE (for working on this repo standalone) - # Currently nothing to do -> no-op. - # So why is this even here? Enables future expansion (e.g to add transitive dependencies) without changing the user interface. - pass + http_archive( + name = "rules_python", + sha256 = "84aec9e21cc56fbc7f1335035a71c850d1b9b5cc6ff497306f84cced9a769841", + strip_prefix = "rules_python-0.23.1", + url = "https://github.com/bazelbuild/rules_python/releases/download/0.23.1/rules_python-0.23.1.tar.gz", + ) + + # TODO: may not be needed, try removing. + BAZEL_SKYLIB_VERSION = "1.4.2" + + http_archive( + name = "bazel_skylib", + sha256 = "66ffd9315665bfaafc96b52278f57c7e2dd09f5ede279ea6d39b2be471e7e3aa", + urls = [ + "https://mirror.bazel.build/github.com/bazelbuild/bazel-skylib/releases/download/{0}/bazel-skylib-{0}.tar.gz".format(BAZEL_SKYLIB_VERSION), + "https://github.com/bazelbuild/bazel-skylib/releases/download/{0}/bazel-skylib-{0}.tar.gz".format(BAZEL_SKYLIB_VERSION), + ], + ) + + install_python_deps() From 4e7836cfb812c1443e6e9eb39304423c302181ec Mon Sep 17 00:00:00 2001 From: Gary Miguel Date: Fri, 16 Jun 2023 22:46:30 +0000 Subject: [PATCH 2/2] I guess working --- WORKSPACE | 34 ++++++++++++++++++++++++++++++++-- python_requirements_lock.bzl | 2 +- requirements.bzl | 0 workspace_setup.bzl | 27 --------------------------- 4 files changed, 33 insertions(+), 30 deletions(-) create mode 100644 requirements.bzl diff --git a/WORKSPACE b/WORKSPACE index 49c9a00..50dab01 100644 --- a/WORKSPACE +++ b/WORKSPACE @@ -4,9 +4,33 @@ workspace(name = "hedron_compile_commands") -load("@hedron_compile_commands//:workspace_setup.bzl", "hedron_compile_commands_setup") +load("@bazel_tools//tools/build_defs/repo:http.bzl", "http_archive") -hedron_compile_commands_setup() +# TODO(cpsauer): move everything above hedron_compile_commands_setup() into setup macros. +BAZEL_SKYLIB_VERSION = "1.4.2" + +http_archive( + name = "bazel_skylib", + sha256 = "66ffd9315665bfaafc96b52278f57c7e2dd09f5ede279ea6d39b2be471e7e3aa", + urls = [ + "https://mirror.bazel.build/github.com/bazelbuild/bazel-skylib/releases/download/{0}/bazel-skylib-{0}.tar.gz".format(BAZEL_SKYLIB_VERSION), + "https://github.com/bazelbuild/bazel-skylib/releases/download/{0}/bazel-skylib-{0}.tar.gz".format(BAZEL_SKYLIB_VERSION), + ], +) + +http_archive( + name = "rules_python", + sha256 = "84aec9e21cc56fbc7f1335035a71c850d1b9b5cc6ff497306f84cced9a769841", + strip_prefix = "rules_python-0.23.1", + url = "https://github.com/bazelbuild/rules_python/releases/download/0.23.1/rules_python-0.23.1.tar.gz", +) + +load("@rules_python//python:repositories.bzl", "python_register_toolchains") + +python_register_toolchains( + name = "python_toolchain", + python_version = "3.11", +) # For re-generating python_requirements_lock.bzl: # * update python_requirements_lock.txt @@ -14,8 +38,14 @@ hedron_compile_commands_setup() # * run `bazel build @pip//...`, # * cp external/pip/requirements.bzl python_requirements_lock.bzl +# load("@python_toolchain//:defs.bzl", "interpreter") # load("@rules_python//python:pip.bzl", "pip_parse") # pip_parse( # name = "pip", +# python_interpreter_target = interpreter, # requirements_lock = "//:python_requirements_lock.txt", # ) + +load("@hedron_compile_commands//:workspace_setup.bzl", "hedron_compile_commands_setup") + +hedron_compile_commands_setup() diff --git a/python_requirements_lock.bzl b/python_requirements_lock.bzl index 8adb5a7..26a20fc 100755 --- a/python_requirements_lock.bzl +++ b/python_requirements_lock.bzl @@ -11,7 +11,7 @@ all_requirements = ["@pip_orjson//:pkg"] all_whl_requirements = ["@pip_orjson//:whl"] _packages = [("pip_orjson", "orjson==3.9.1")] -_config = {"download_only": False, "enable_implicit_namespace_pkgs": False, "environment": {}, "extra_pip_args": [], "isolated": True, "pip_data_exclude": [], "python_interpreter": "python3", "quiet": True, "repo": "pip", "repo_prefix": "pip_", "timeout": 600} +_config = {"download_only": False, "enable_implicit_namespace_pkgs": False, "environment": {}, "extra_pip_args": [], "isolated": True, "pip_data_exclude": [], "python_interpreter": "python3", "python_interpreter_target": "@python_toolchain_x86_64-unknown-linux-gnu//:bin/python3", "quiet": True, "repo": "pip", "repo_prefix": "pip_", "timeout": 600} _annotations = {} def _clean_name(name): diff --git a/requirements.bzl b/requirements.bzl new file mode 100644 index 0000000..e69de29 diff --git a/workspace_setup.bzl b/workspace_setup.bzl index cd09c1c..98f1798 100644 --- a/workspace_setup.bzl +++ b/workspace_setup.bzl @@ -2,15 +2,8 @@ # Do not change the filename; it is part of the user interface. -load("@bazel_tools//tools/build_defs/repo:http.bzl", _http_archive = "http_archive") -load("@bazel_tools//tools/build_defs/repo:utils.bzl", "maybe") - -# TODO: This fails to load because it relies on rules_python already being defined. load("//:python_requirements_lock.bzl", install_python_deps = "install_deps") -def http_archive(**kwargs): - maybe(_http_archive, **kwargs) - def hedron_compile_commands_setup(): """Set up a WORKSPACE to have hedron_compile_commands used within it.""" @@ -18,24 +11,4 @@ def hedron_compile_commands_setup(): # See invocations in: # README.md (for users) # WORKSPACE (for working on this repo standalone) - - http_archive( - name = "rules_python", - sha256 = "84aec9e21cc56fbc7f1335035a71c850d1b9b5cc6ff497306f84cced9a769841", - strip_prefix = "rules_python-0.23.1", - url = "https://github.com/bazelbuild/rules_python/releases/download/0.23.1/rules_python-0.23.1.tar.gz", - ) - - # TODO: may not be needed, try removing. - BAZEL_SKYLIB_VERSION = "1.4.2" - - http_archive( - name = "bazel_skylib", - sha256 = "66ffd9315665bfaafc96b52278f57c7e2dd09f5ede279ea6d39b2be471e7e3aa", - urls = [ - "https://mirror.bazel.build/github.com/bazelbuild/bazel-skylib/releases/download/{0}/bazel-skylib-{0}.tar.gz".format(BAZEL_SKYLIB_VERSION), - "https://github.com/bazelbuild/bazel-skylib/releases/download/{0}/bazel-skylib-{0}.tar.gz".format(BAZEL_SKYLIB_VERSION), - ], - ) - install_python_deps()