Get the arguments emcc would pass to clang

Author: kon72

BUILD

@@ -25,4 +25,13 @@ filegroup(
 # Implementation:
 # If you are looking into the implementation, start with the overview in ImplementationReadme.md.
 
-exports_files(["refresh.template.py", "check_python_version.template.py"])  # For implicit use by the refresh_compile_commands macro, not direct use.
+exports_files(["refresh.template.py", "check_python_version.template.py", "print_args.py"])  # For implicit use by the refresh_compile_commands macro, not direct use.
+
+filegroup(
+    name = "print_args",
+    srcs = select({
+        "@bazel_tools//src/conditions:host_windows": [":print_args.bat"],
+        "//conditions:default": [":print_args.sh"],
+    }),
+    visibility = ["//visibility:public"],
+)

print_args.bat

@@ -0,0 +1,3 @@
+@ECHO OFF
+
+py -3 %HEDRON_COMPILE_COMMANDS_PRINT_ARGS_PY% %*

print_args.py

@@ -0,0 +1,19 @@
+"""
+Prints the arguments passed to the script
+"""
+
+import sys
+
+
+def main():
+  print('===HEDRON_COMPILE_COMMANDS_BEGIN_ARGS===')
+  for arg in sys.argv[1:]:
+    print(arg)
+  print('===HEDRON_COMPILE_COMMANDS_END_ARGS===')
+
+  # We purposely return a non-zero exit code to have the emcc process exit after running this fake clang wrapper.
+  sys.exit(1)
+
+
+if __name__ == '__main__':
+  main()

print_args.sh

@@ -0,0 +1,3 @@
+#!/bin/bash
+
+exec python3 "${HEDRON_COMPILE_COMMANDS_PRINT_ARGS_PY}" "$@"

refresh.template.py

@@ -283,17 +283,6 @@ def _get_headers_gcc(compile_args: typing.List[str], source_path: str, action_ke
     return headers, should_cache
 
 
-@functools.lru_cache(maxsize=None)
-def _get_clang_or_gcc():
-    """Returns clang or gcc, if you have one of them on your path."""
-    if shutil.which('clang'):
-        return 'clang'
-    elif shutil.which('gcc'):
-        return 'gcc'
-    else:
-        return None
-
-
 def windows_list2cmdline(seq):
     """
     Copied from list2cmdline in https://github.com/python/cpython/blob/main/Lib/subprocess.py because we need it but it's not exported as part of the public API.
@@ -567,18 +556,7 @@ def _get_headers(compile_action, source_path: str):
     if compile_action.arguments[0].endswith('cl.exe'): # cl.exe and also clang-cl.exe
         headers, should_cache = _get_headers_msvc(compile_action.arguments, source_path)
     else:
-        # Emscripten is tricky. There isn't an easy way to make it emcc run without lots of environment variables.
-        # So...rather than doing our usual script unwrapping, we just swap in clang/gcc and use that to get headers, knowing that they'll accept the same argument format.
-            # You can unwrap emcc.sh to emcc.py via next(pathlib.Path('external').glob('emscripten_bin_*/emscripten/emcc.py')).as_posix()
-            # But then the underlying emcc needs a configuration file that itself depends on lots of environment variables.
-            # If we ever pick this back up, note that you can supply that config via compile_args += ["--em-config", "external/emsdk/emscripten_toolchain/emscripten_config"]
-        args = compile_action.arguments
-        if args[0].endswith('emcc.sh') or args[0].endswith('emcc.bat'):
-            alternate_compiler = _get_clang_or_gcc()
-            if not alternate_compiler: return set() # Skip getting headers.
-            args = args.copy()
-            args[0] = alternate_compiler
-        headers, should_cache = _get_headers_gcc(args, source_path, compile_action.actionKey)
+        headers, should_cache = _get_headers_gcc(compile_action.arguments, source_path, compile_action.actionKey)
 
     # Cache for future use
     if output_file and should_cache:
@@ -767,6 +745,73 @@ def _apple_platform_patch(compile_args: typing.List[str]):
     return compile_args
 
 
+def _get_sysroot(args: typing.List[str]):
+    for idx, arg in enumerate(args):
+        if arg == '--sysroot' or arg == '-isysroot':
+            if idx + 1 < len(args):
+                return pathlib.PurePath(args[idx + 1])
+        elif arg.startswith('--sysroot='):
+            return pathlib.PurePath(arg[len('--sysroot='):])
+        elif arg.startswith('-isysroot'):
+            return pathlib.PurePath(arg[len('-isysroot'):])
+    return None
+
+
+def _emscripten_platform_patch(compile_args: typing.List[str]):
+    """De-Bazel the command into something clangd can parse.
+
+    This function has fixes specific to Emscripten platforms, but you should call it on all platforms. It'll determine whether the fixes should be applied or not
+    """
+    emcc_driver = pathlib.Path(compile_args[0])
+    if emcc_driver.name != 'emcc.sh' and emcc_driver.name != 'emcc.bat':
+        return compile_args
+
+    workspace_absolute = pathlib.PurePath(os.environ["BUILD_WORKSPACE_DIRECTORY"])
+    sysroot = _get_sysroot(compile_args)
+    assert sysroot, f'Emscripten sysroot not detected in CMD: {compile_args}'
+
+    def get_workspace_root(path_from_execroot: pathlib.PurePath):
+        assert path_from_execroot.parts[0] == 'external'
+        return pathlib.PurePath('external') / path_from_execroot.parts[1]
+
+    environment = {
+        'EXT_BUILD_ROOT': str(workspace_absolute),
+        'EM_BIN_PATH': str(get_workspace_root(sysroot)),
+        'EM_CONFIG_PATH': str(get_workspace_root(emcc_driver) / 'emscripten_toolchain' / 'emscripten_config'),
+        'EMCC_SKIP_SANITY_CHECK': '1',
+        'EM_COMPILER_WRAPPER': str(pathlib.PurePath({print_args_executable})),
+        'HEDRON_COMPILE_COMMANDS_PRINT_ARGS_PY': str(pathlib.PurePath({print_args_py})),
+        'PATH': os.environ['PATH'],
+    }
+
+    # We run the emcc process with the environment variable EM_COMPILER_WRAPPER to intercept the command line arguments passed to `clang`.
+    emcc_process = subprocess.run(
+        [emcc_driver] + compile_args[1:],
+        # MIN_PY=3.7: Replace PIPEs with capture_output.
+        stdout=subprocess.PIPE,
+        stderr=subprocess.PIPE,
+        env=environment,
+        encoding=locale.getpreferredencoding(),
+        check=False, # We explicitly ignore errors and carry on.
+    )
+
+    lines = emcc_process.stdout.splitlines()
+
+    # Parse the arguments from the output of the emcc process.
+    if BEGIN_ARGS_MARKER in lines:
+        begin_args_idx = lines.index(BEGIN_ARGS_MARKER)
+        end_args_idx = lines.index(END_ARGS_MARKER, begin_args_idx + 1)
+        args = lines[begin_args_idx + 1:end_args_idx]
+        clang_driver = pathlib.PurePath(args[0])
+        if _is_relative_to(clang_driver, workspace_absolute):
+            args[0] = clang_driver.relative_to(workspace_absolute).as_posix()
+        return args
+
+    assert False, f'Failed to parse emcc output: {emcc_process.stderr}'
+BEGIN_ARGS_MARKER = '===HEDRON_COMPILE_COMMANDS_BEGIN_ARGS==='
+END_ARGS_MARKER = '===HEDRON_COMPILE_COMMANDS_END_ARGS==='
+
+
 def _all_platform_patch(compile_args: typing.List[str]):
     """Apply de-Bazeling fixes to the compile command that are shared across target platforms."""
     # clangd writes module cache files to the wrong place
@@ -1020,6 +1065,7 @@ def _get_cpp_command_for_files(compile_action):
     # Patch command by platform
     compile_action.arguments = _all_platform_patch(compile_action.arguments)
     compile_action.arguments = _apple_platform_patch(compile_action.arguments)
+    compile_action.arguments = _emscripten_platform_patch(compile_action.arguments)
     # Android and Linux and grailbio LLVM toolchains: Fine as is; no special patching needed.
 
     source_files, header_files = _get_files(compile_action)

refresh_compile_commands.bzl

@@ -98,6 +98,7 @@ def refresh_compile_commands(
             version_checker_script_name,
             script_name,
         ],
+        data = ["@hedron_compile_commands//:print_args"],
         imports = [''], # Allows binary to import templated script, even if this macro is being called inside a sub package. See https://github.com/hedronvision/bazel-compile-commands-extractor/issues/137
         **kwargs
     )
@@ -115,6 +116,8 @@ def _expand_template_impl(ctx):
             "        {windows_default_include_paths}": "\n".join(["        %r," % path for path in find_cpp_toolchain(ctx).built_in_include_directories]),  # find_cpp_toolchain is from https://docs.bazel.build/versions/main/integrating-with-rules-cc.html
             "{exclude_headers}": repr(ctx.attr.exclude_headers),
             "{exclude_external_sources}": repr(ctx.attr.exclude_external_sources),
+            "{print_args_executable}": repr(ctx.executable._print_args_executable.path),
+            "{print_args_py}": repr(ctx.file._print_args_py.path),
         },
     )
     return DefaultInfo(files = depset([script]))
@@ -125,6 +128,8 @@ _expand_template = rule(
         "exclude_external_sources": attr.bool(default = False),
         "exclude_headers": attr.string(values = ["all", "external", ""]),  # "" needed only for compatibility with Bazel < 3.6.0
         "_script_template": attr.label(allow_single_file = True, default = "refresh.template.py"),
+        "_print_args_executable": attr.label(executable = True, allow_single_file = True, cfg = "target", default = "print_args"),
+        "_print_args_py": attr.label(allow_single_file = True, default = "print_args.py"),
         # For Windows INCLUDE. If this were eliminated, for example by the resolution of https://github.com/clangd/clangd/issues/123, we'd be able to just use a macro and skylib's expand_template rule: https://github.com/bazelbuild/bazel-skylib/pull/330
         # Once https://github.com/bazelbuild/bazel/pull/17108 is widely released, we should be able to eliminate this and get INCLUDE directly. Perhaps for 7.0? Should be released in the sucessor to 6.0
         "_cc_toolchain": attr.label(default = "@bazel_tools//tools/cpp:current_cc_toolchain"),