extend bootstrap for PGO on windows

When building LLVM/LLD as part of a build that asks LLVM to generate profiles, e.g. when
doing PGO, cmake or clang-cl don't automatically link clang's profiler runtime in,
causing undefined reference errors at link-time.

We do that manually, by adding clang's resource library folder to the library search path:
- for LLVM itself, by extending the linker args that `rustc_llvm`'s build script
  uses, to avoid the linker errors when linking `rustc_driver`.
- for LLD, by extending cmake's linker flags during the LLD build step.

Author: lqd

Diff for: src/bootstrap/compile.rs

@@ -25,6 +25,7 @@ use crate::config::{LlvmLibunwind, TargetSelection};
 use crate::dist;
 use crate::native;
 use crate::tool::SourceType;
+use crate::util::get_clang_cl_resource_dir;
 use crate::util::{exe, is_debug_info, is_dylib, output, symlink_dir, t, up_to_date};
 use crate::LLVM_TOOLS;
 use crate::{CLang, Compiler, DependencyType, GitRepo, Mode};
@@ -769,10 +770,38 @@ pub fn rustc_cargo_env(builder: &Builder<'_>, cargo: &mut Cargo, target: TargetS
         if let Some(s) = target_config.and_then(|c| c.llvm_config.as_ref()) {
             cargo.env("CFG_LLVM_ROOT", s);
         }
-        // Some LLVM linker flags (-L and -l) may be needed to link rustc_llvm.
+
+        // Some LLVM linker flags (-L and -l) may be needed to link `rustc_llvm`. Its build script
+        // expects these to be passed via the `LLVM_LINKER_FLAGS` env variable, separated by
+        // whitespace.
+        //
+        // For example:
+        // - on windows, when `clang-cl` is used with instrumentation, we need to manually add
+        // clang's runtime library resource directory so that the profiler runtime library can be
+        // found. This is to avoid the linker errors about undefined references to
+        // `__llvm_profile_instrument_memop` when linking `rustc_driver`.
+        let mut llvm_linker_flags = String::new();
+        if builder.config.llvm_profile_generate && target.contains("msvc") {
+            if let Some(ref clang_cl_path) = builder.config.llvm_clang_cl {
+                // Add clang's runtime library directory to the search path
+                let clang_rt_dir = get_clang_cl_resource_dir(clang_cl_path);
+                llvm_linker_flags.push_str(&format!("-L{}", clang_rt_dir.display()));
+            }
+        }
+
+        // The config can also specify its own llvm linker flags.
         if let Some(ref s) = builder.config.llvm_ldflags {
-            cargo.env("LLVM_LINKER_FLAGS", s);
+            if !llvm_linker_flags.is_empty() {
+                llvm_linker_flags.push_str(" ");
+            }
+            llvm_linker_flags.push_str(s);
+        }
+
+        // Set the linker flags via the env var that `rustc_llvm`'s build script will read.
+        if !llvm_linker_flags.is_empty() {
+            cargo.env("LLVM_LINKER_FLAGS", llvm_linker_flags);
         }
+
         // Building with a static libstdc++ is only supported on linux right now,
         // not for MSVC or macOS
         if builder.config.llvm_static_stdcpp

Diff for: src/bootstrap/native.rs

@@ -18,6 +18,7 @@ use std::process::Command;
 
 use crate::builder::{Builder, RunConfig, ShouldRun, Step};
 use crate::config::TargetSelection;
+use crate::util::get_clang_cl_resource_dir;
 use crate::util::{self, exe, output, program_out_of_date, t, up_to_date};
 use crate::{CLang, GitRepo};
 
@@ -755,7 +756,22 @@ impl Step for Lld {
         t!(fs::create_dir_all(&out_dir));
 
         let mut cfg = cmake::Config::new(builder.src.join("src/llvm-project/lld"));
-        configure_cmake(builder, target, &mut cfg, true, LdFlags::default());
+        let mut ldflags = LdFlags::default();
+
+        // When building LLD as part of a build with instrumentation on windows, for example
+        // when doing PGO on CI, cmake or clang-cl don't automatically link clang's
+        // profiler runtime in. In that case, we need to manually ask cmake to do it, to avoid
+        // linking errors, much like LLVM's cmake setup does in that situation.
+        if builder.config.llvm_profile_generate && target.contains("msvc") {
+            if let Some(clang_cl_path) = builder.config.llvm_clang_cl.as_ref() {
+                // Find clang's runtime library directory and push that as a search path to the
+                // cmake linker flags.
+                let clang_rt_dir = get_clang_cl_resource_dir(clang_cl_path);
+                ldflags.push_all(&format!("/libpath:{}", clang_rt_dir.display()));
+            }
+        }
+
+        configure_cmake(builder, target, &mut cfg, true, ldflags);
 
         // This is an awful, awful hack. Discovered when we migrated to using
         // clang-cl to compile LLVM/LLD it turns out that LLD, when built out of

Diff for: src/bootstrap/util.rs

@@ -576,3 +576,27 @@ fn absolute_windows(path: &std::path::Path) -> std::io::Result<std::path::PathBu
         }
     }
 }
+
+/// Adapted from https://github.com/llvm/llvm-project/blob/782e91224601e461c019e0a4573bbccc6094fbcd/llvm/cmake/modules/HandleLLVMOptions.cmake#L1058-L1079
+///
+/// When `clang-cl` is used with instrumentation, we need to add clang's runtime library resource
+/// directory to the linker flags, otherwise there will be linker errors about the profiler runtime
+/// missing. This function returns the path to that directory.
+pub fn get_clang_cl_resource_dir(clang_cl_path: &str) -> PathBuf {
+    // Similar to how LLVM does it, to find clang's library runtime directory:
+    // - we ask `clang-cl` to locate the `clang_rt.builtins` lib.
+    let mut builtins_locator = Command::new(clang_cl_path);
+    builtins_locator.args(&["/clang:-print-libgcc-file-name", "/clang:--rtlib=compiler-rt"]);
+
+    let clang_rt_builtins = output(&mut builtins_locator);
+    let clang_rt_builtins = Path::new(clang_rt_builtins.trim());
+    assert!(
+        clang_rt_builtins.exists(),
+        "`clang-cl` must correctly locate the library runtime directory"
+    );
+
+    // - the profiler runtime will be located in the same directory as the builtins lib, like
+    // `$LLVM_DISTRO_ROOT/lib/clang/$LLVM_VERSION/lib/windows`.
+    let clang_rt_dir = clang_rt_builtins.parent().expect("The clang lib folder should exist");
+    clang_rt_dir.to_path_buf()
+}