Auto merge of rust-lang#120348 - bjorn3:per_target_backend_selection, r=albertlarsan68

Support configuring the set of codegen backends to build per host triple

This allows building the compiler itself with one backend while using another backend at runtime. For example this allows compiling rustc to wasm using LLVM, while using Cranelift at runtime to produce actual code. Cranelift can't compile to wasm, but is perfectly capable of running on wasm. LLVM can compile to wasm, but can't run on wasm. [^1]

[^1]: The prototype of this still requires a couple of other patches.

Author: bors

config.example.toml

@@ -829,6 +829,11 @@
 # target triples containing `-none`, `nvptx`, `switch`, or `-uefi`.
 #no-std = <platform-specific> (bool)
 
+# This is an array of the codegen backends that will be compiled a rustc
+# compiled for this target, overriding the global rust.codegen-backends option.
+# See that option for more info.
+#codegen-backends = rust.codegen-backends (array)
+
 # =============================================================================
 # Distribution options
 #

src/bootstrap/src/core/build_steps/compile.rs

@@ -1039,7 +1039,7 @@ impl Step for Rustc {
 pub fn rustc_cargo(builder: &Builder<'_>, cargo: &mut Cargo, target: TargetSelection, stage: u32) {
     cargo
         .arg("--features")
-        .arg(builder.rustc_features(builder.kind))
+        .arg(builder.rustc_features(builder.kind, target))
         .arg("--manifest-path")
         .arg(builder.src.join("compiler/rustc/Cargo.toml"));
 
@@ -1096,7 +1096,7 @@ pub fn rustc_cargo_env(
         cargo.env("CFG_OMIT_GIT_HASH", "1");
     }
 
-    if let Some(backend) = builder.config.default_codegen_backend() {
+    if let Some(backend) = builder.config.default_codegen_backend(target) {
         cargo.env("CFG_DEFAULT_CODEGEN_BACKEND", backend);
     }
 
@@ -1137,7 +1137,7 @@ pub fn rustc_cargo_env(
     // build. If we are in a check build we still go ahead here presuming we've
     // detected that LLVM is already built and good to go which helps prevent
     // busting caches (e.g. like #71152).
-    if builder.config.llvm_enabled() {
+    if builder.config.llvm_enabled(target) {
         let building_is_expensive =
             crate::core::build_steps::llvm::prebuilt_llvm_config(builder, target).is_err();
         // `top_stage == stage` might be false for `check --stage 1`, if we are building the stage 1 compiler
@@ -1281,7 +1281,7 @@ pub(crate) const CODEGEN_BACKEND_PREFIX: &str = "rustc_codegen_";
 fn is_codegen_cfg_needed(path: &TaskPath, run: &RunConfig<'_>) -> bool {
     if path.path.to_str().unwrap().contains(&CODEGEN_BACKEND_PREFIX) {
         let mut needs_codegen_backend_config = true;
-        for &backend in &run.builder.config.rust_codegen_backends {
+        for &backend in run.builder.config.codegen_backends(run.target) {
             if path
                 .path
                 .to_str()
@@ -1318,7 +1318,7 @@ impl Step for CodegenBackend {
             return;
         }
 
-        for &backend in &run.builder.config.rust_codegen_backends {
+        for &backend in run.builder.config.codegen_backends(run.target) {
             if backend == "llvm" {
                 continue; // Already built as part of rustc
             }
@@ -1425,7 +1425,7 @@ fn copy_codegen_backends_to_sysroot(
         return;
     }
 
-    for backend in builder.config.rust_codegen_backends.iter() {
+    for backend in builder.config.codegen_backends(target) {
         if backend == "llvm" {
             continue; // Already built as part of rustc
         }
@@ -1732,7 +1732,7 @@ impl Step for Assemble {
         // to not fail while linking the artifacts.
         build_compiler.stage = actual_stage;
 
-        for &backend in builder.config.rust_codegen_backends.iter() {
+        for &backend in builder.config.codegen_backends(target_compiler.host) {
             if backend == "llvm" {
                 continue; // Already built as part of rustc
             }
@@ -1817,7 +1817,7 @@ impl Step for Assemble {
             }
         }
 
-        if builder.config.rust_codegen_backends.contains(&INTERNER.intern_str("llvm")) {
+        if builder.config.llvm_enabled(target_compiler.host) {
             let llvm::LlvmResult { llvm_config, .. } =
                 builder.ensure(llvm::Llvm { target: target_compiler.host });
             if !builder.config.dry_run() && builder.config.llvm_tools_enabled {

src/bootstrap/src/core/build_steps/dist.rs

@@ -1278,7 +1278,7 @@ impl Step for CodegenBackend {
     }
 
     fn make_run(run: RunConfig<'_>) {
-        for &backend in &run.builder.config.rust_codegen_backends {
+        for &backend in run.builder.config.codegen_backends(run.target) {
             if backend == "llvm" {
                 continue; // Already built as part of rustc
             }
@@ -1302,7 +1302,7 @@ impl Step for CodegenBackend {
             return None;
         }
 
-        if !builder.config.rust_codegen_backends.contains(&self.backend) {
+        if !builder.config.codegen_backends(self.compiler.host).contains(&self.backend) {
             return None;
         }
 

src/bootstrap/src/core/build_steps/test.rs

@@ -1798,7 +1798,7 @@ NOTE: if you're sure you want to do this, please open an issue as to why. In the
 
         let mut llvm_components_passed = false;
         let mut copts_passed = false;
-        if builder.config.llvm_enabled() {
+        if builder.config.llvm_enabled(compiler.host) {
             let llvm::LlvmResult { llvm_config, .. } =
                 builder.ensure(llvm::Llvm { target: builder.config.build });
             if !builder.config.dry_run() {
@@ -3121,7 +3121,8 @@ impl Step for CodegenCranelift {
             return;
         }
 
-        if !builder.config.rust_codegen_backends.contains(&INTERNER.intern_str("cranelift")) {
+        if !builder.config.codegen_backends(run.target).contains(&INTERNER.intern_str("cranelift"))
+        {
             builder.info("cranelift not in rust.codegen-backends. skipping");
             return;
         }
@@ -3245,7 +3246,7 @@ impl Step for CodegenGCC {
             return;
         }
 
-        if !builder.config.rust_codegen_backends.contains(&INTERNER.intern_str("gcc")) {
+        if !builder.config.codegen_backends(run.target).contains(&INTERNER.intern_str("gcc")) {
             builder.info("gcc not in rust.codegen-backends. skipping");
             return;
         }

src/bootstrap/src/core/builder.rs

@@ -1229,7 +1229,7 @@ impl<'a> Builder<'a> {
     /// Note that this returns `None` if LLVM is disabled, or if we're in a
     /// check build or dry-run, where there's no need to build all of LLVM.
     fn llvm_config(&self, target: TargetSelection) -> Option<PathBuf> {
-        if self.config.llvm_enabled() && self.kind != Kind::Check && !self.config.dry_run() {
+        if self.config.llvm_enabled(target) && self.kind != Kind::Check && !self.config.dry_run() {
             let llvm::LlvmResult { llvm_config, .. } = self.ensure(llvm::Llvm { target });
             if llvm_config.is_file() {
                 return Some(llvm_config);
@@ -1991,7 +1991,8 @@ impl<'a> Builder<'a> {
             };
 
             if let Some(limit) = limit {
-                if stage == 0 || self.config.default_codegen_backend().unwrap_or_default() == "llvm"
+                if stage == 0
+                    || self.config.default_codegen_backend(target).unwrap_or_default() == "llvm"
                 {
                     rustflags.arg(&format!("-Cllvm-args=-import-instr-limit={limit}"));
                 }

src/bootstrap/src/core/config/config.rs

@@ -577,6 +577,7 @@ pub struct Target {
     pub wasi_root: Option<PathBuf>,
     pub qemu_rootfs: Option<PathBuf>,
     pub no_std: bool,
+    pub codegen_backends: Option<Vec<Interned<String>>>,
 }
 
 impl Target {
@@ -1135,6 +1136,7 @@ define_config! {
         wasi_root: Option<String> = "wasi-root",
         qemu_rootfs: Option<String> = "qemu-rootfs",
         no_std: Option<bool> = "no-std",
+        codegen_backends: Option<Vec<String>> = "codegen-backends",
     }
 }
 
@@ -1840,6 +1842,24 @@ impl Config {
                 target.profiler = cfg.profiler;
                 target.rpath = cfg.rpath;
 
+                if let Some(ref backends) = cfg.codegen_backends {
+                    let available_backends = vec!["llvm", "cranelift", "gcc"];
+
+                    target.codegen_backends = Some(backends.iter().map(|s| {
+                        if let Some(backend) = s.strip_prefix(CODEGEN_BACKEND_PREFIX) {
+                            if available_backends.contains(&backend) {
+                                panic!("Invalid value '{s}' for 'target.{triple}.codegen-backends'. Instead, please use '{backend}'.");
+                            } else {
+                                println!("HELP: '{s}' for 'target.{triple}.codegen-backends' might fail. \
+                                    Codegen backends are mostly defined without the '{CODEGEN_BACKEND_PREFIX}' prefix. \
+                                    In this case, it would be referred to as '{backend}'.");
+                            }
+                        }
+
+                        INTERNER.intern_str(s)
+                    }).collect());
+                }
+
                 config.target_config.insert(TargetSelection::from_user(&triple), target);
             }
         }
@@ -2222,8 +2242,8 @@ impl Config {
         self.target_config.get(&target).map(|t| t.rpath).flatten().unwrap_or(self.rust_rpath)
     }
 
-    pub fn llvm_enabled(&self) -> bool {
-        self.rust_codegen_backends.contains(&INTERNER.intern_str("llvm"))
+    pub fn llvm_enabled(&self, target: TargetSelection) -> bool {
+        self.codegen_backends(target).contains(&INTERNER.intern_str("llvm"))
     }
 
     pub fn llvm_libunwind(&self, target: TargetSelection) -> LlvmLibunwind {
@@ -2242,8 +2262,15 @@ impl Config {
         self.submodules.unwrap_or(rust_info.is_managed_git_subrepository())
     }
 
-    pub fn default_codegen_backend(&self) -> Option<Interned<String>> {
-        self.rust_codegen_backends.get(0).cloned()
+    pub fn codegen_backends(&self, target: TargetSelection) -> &[Interned<String>] {
+        self.target_config
+            .get(&target)
+            .and_then(|cfg| cfg.codegen_backends.as_deref())
+            .unwrap_or(&self.rust_codegen_backends)
+    }
+
+    pub fn default_codegen_backend(&self, target: TargetSelection) -> Option<Interned<String>> {
+        self.codegen_backends(target).get(0).cloned()
     }
 
     pub fn git_config(&self) -> GitConfig<'_> {

src/bootstrap/src/core/sanity.rs

@@ -16,7 +16,6 @@ use std::path::PathBuf;
 use std::process::Command;
 
 use crate::core::config::Target;
-use crate::utils::cache::INTERNER;
 use crate::utils::helpers::output;
 use crate::Build;
 
@@ -88,19 +87,19 @@ pub fn check(build: &mut Build) {
     }
 
     // We need cmake, but only if we're actually building LLVM or sanitizers.
-    let building_llvm = build.config.rust_codegen_backends.contains(&INTERNER.intern_str("llvm"))
-        && build
-            .hosts
-            .iter()
-            .map(|host| {
-                build
+    let building_llvm = build
+        .hosts
+        .iter()
+        .map(|host| {
+            build.config.llvm_enabled(*host)
+                && build
                     .config
                     .target_config
                     .get(host)
                     .map(|config| config.llvm_config.is_none())
                     .unwrap_or(true)
-            })
-            .any(|build_llvm_ourselves| build_llvm_ourselves);
+        })
+        .any(|build_llvm_ourselves| build_llvm_ourselves);
 
     let need_cmake = building_llvm || build.config.any_sanitizers_to_build();
     if need_cmake && cmd_finder.maybe_have("cmake").is_none() {
@@ -190,13 +189,16 @@ than building it.
         if !build.config.dry_run() {
             cmd_finder.must_have(build.cxx(*host).unwrap());
         }
-    }
 
-    if build.config.rust_codegen_backends.contains(&INTERNER.intern_str("llvm")) {
-        // Externally configured LLVM requires FileCheck to exist
-        let filecheck = build.llvm_filecheck(build.build);
-        if !filecheck.starts_with(&build.out) && !filecheck.exists() && build.config.codegen_tests {
-            panic!("FileCheck executable {filecheck:?} does not exist");
+        if build.config.llvm_enabled(*host) {
+            // Externally configured LLVM requires FileCheck to exist
+            let filecheck = build.llvm_filecheck(build.build);
+            if !filecheck.starts_with(&build.out)
+                && !filecheck.exists()
+                && build.config.codegen_tests
+            {
+                panic!("FileCheck executable {filecheck:?} does not exist");
+            }
         }
     }
 

src/bootstrap/src/lib.rs

@@ -731,12 +731,12 @@ impl Build {
     }
 
     /// Gets the space-separated set of activated features for the compiler.
-    fn rustc_features(&self, kind: Kind) -> String {
+    fn rustc_features(&self, kind: Kind, target: TargetSelection) -> String {
         let mut features = vec![];
         if self.config.jemalloc {
             features.push("jemalloc");
         }
-        if self.config.llvm_enabled() || kind == Kind::Check {
+        if self.config.llvm_enabled(target) || kind == Kind::Check {
             features.push("llvm");
         }
         // keep in sync with `bootstrap/compile.rs:rustc_cargo_env`
@@ -1561,7 +1561,8 @@ impl Build {
                         || target
                             .map(|t| self.config.profiler_enabled(t))
                             .unwrap_or_else(|| self.config.any_profiler_enabled()))
-                    && (dep != "rustc_codegen_llvm" || self.config.llvm_enabled())
+                    && (dep != "rustc_codegen_llvm"
+                        || self.config.hosts.iter().any(|host| self.config.llvm_enabled(*host)))
                 {
                     list.push(*dep);
                 }

src/bootstrap/src/utils/change_tracker.rs

@@ -114,4 +114,9 @@ pub const CONFIG_CHANGE_HISTORY: &[ChangeInfo] = &[
         severity: ChangeSeverity::Warning,
         summary: "A new `optimized-compiler-builtins` option has been introduced. Whether to build llvm's `compiler-rt` from source is no longer implicitly controlled by git state. See the PR for more details.",
     },
+    ChangeInfo {
+        change_id: 120348,
+        severity: ChangeSeverity::Info,
+        summary: "New option `target.<triple>.codegen-backends` added to config.toml.",
+    },
 ];