Introduce dedicated -Zdylib-lto flag for enabling LTO on dylibs

Author: Kobzol

Diff for: compiler/rustc_codegen_llvm/src/back/lto.rs

@@ -82,10 +82,24 @@ fn prepare_lto(
                 );
                 return Err(e);
             } else if *crate_type == CrateType::Dylib {
-                diag_handler.warn("LTO with dylibs may not be as effective");
+                if !cgcx.opts.unstable_opts.dylib_lto {
+                    return Err(diag_handler
+                        .fatal("lto cannot be used for `dylib` crate type without `-Zdylib-lto`"));
+                }
             }
         }
 
+        if cgcx.opts.cg.prefer_dynamic && !cgcx.opts.unstable_opts.dylib_lto {
+            diag_handler
+                .struct_err("cannot prefer dynamic linking when performing LTO")
+                .note(
+                    "only 'staticlib', 'bin', and 'cdylib' outputs are \
+                               supported with LTO",
+                )
+                .emit();
+            return Err(FatalError);
+        }
+
         for &(cnum, ref path) in cgcx.each_linked_rlib_for_lto.iter() {
             let exported_symbols =
                 cgcx.exported_symbols.as_ref().expect("needs exported symbols for LTO");

Diff for: compiler/rustc_codegen_ssa/src/back/link.rs

@@ -39,6 +39,7 @@ use cc::windows_registry;
 use regex::Regex;
 use tempfile::Builder as TempFileBuilder;
 
+use itertools::Itertools;
 use std::borrow::Borrow;
 use std::cell::OnceCell;
 use std::collections::BTreeSet;
@@ -49,7 +50,6 @@ use std::ops::Deref;
 use std::path::{Path, PathBuf};
 use std::process::{ExitStatus, Output, Stdio};
 use std::{env, fmt, fs, io, mem, str};
-use itertools::Itertools;
 
 pub fn ensure_removed(diag_handler: &Handler, path: &Path) {
     if let Err(e) = fs::remove_file(path) {
@@ -219,10 +219,15 @@ pub fn each_linked_rlib(
     let lto_active = matches!(sess.lto(), Lto::Fat | Lto::Thin);
     if lto_active {
         for combination in info.dependency_formats.iter().combinations(2) {
-            let (ty1, list1) = combination[0];
-            let (ty2, list2) = combination[1];
+            let (ty1, list1) = &combination[0];
+            let (ty2, list2) = &combination[1];
             if list1 != list2 {
-                return Err(format!("{ty1:?} and {ty2:?} do not have equivalent dependency formats (`{list1:?}` vs `{list2:?}`)"));
+                return Err(errors::LinkRlibError::IncompatibleDependencyFormats {
+                    ty1: format!("{ty1:?}"),
+                    ty2: format!("{ty2:?}"),
+                    list1: format!("{list1:?}"),
+                    list2: format!("{list2:?}"),
+                });
             }
         }
     }

Diff for: compiler/rustc_codegen_ssa/src/errors.rs

@@ -127,6 +127,9 @@ pub enum LinkRlibError {
 
     #[diag(codegen_ssa_rlib_not_found)]
     NotFound { crate_name: Symbol },
+
+    #[diag(codegen_ssa_rlib_incompatible_dependency_formats)]
+    IncompatibleDependencyFormats { ty1: String, ty2: String, list1: String, list2: String },
 }
 
 pub struct ThorinErrorWrapper(pub thorin::Error);

Diff for: compiler/rustc_error_messages/locales/en-US/codegen_ssa.ftl

@@ -34,6 +34,8 @@ codegen_ssa_rlib_only_rmeta_found = could not find rlib for: `{$crate_name}`, fo
 
 codegen_ssa_rlib_not_found = could not find rlib for: `{$crate_name}`
 
+codegen_ssa_rlib_incompatible_dependency_formats = `{$ty1}` and `{$ty2}` do not have equivalent dependency formats (`{$list1}` vs `{$list2}`)
+
 codegen_ssa_linking_failed = linking with `{$linker_path}` failed: {$exit_status}
 
 codegen_ssa_extern_funcs_not_found = some `extern` functions couldn't be found; some native libraries may need to be installed or have their path specified

Diff for: compiler/rustc_interface/src/tests.rs

@@ -648,6 +648,7 @@ fn test_unstable_options_tracking_hash() {
     untracked!(dump_mir_dir, String::from("abc"));
     untracked!(dump_mir_exclude_pass_number, true);
     untracked!(dump_mir_graphviz, true);
+    untracked!(dylib_lto, true);
     untracked!(emit_stack_sizes, true);
     untracked!(future_incompat_test, true);
     untracked!(hir_stats, true);

Diff for: compiler/rustc_session/src/options.rs

@@ -1295,6 +1295,8 @@ options! {
         an additional `.html` file showing the computed coverage spans."),
     dwarf_version: Option<u32> = (None, parse_opt_number, [TRACKED],
         "version of DWARF debug information to emit (default: 2 or 4, depending on platform)"),
+    dylib_lto: bool = (false, parse_bool, [UNTRACKED],
+        "enables LTO for dylib crate type"),
     emit_stack_sizes: bool = (false, parse_bool, [UNTRACKED],
         "emit a section containing stack size metadata (default: no)"),
     emit_thin_lto: bool = (true, parse_bool, [TRACKED],

Diff for: config.toml.example

@@ -638,10 +638,10 @@ changelog-seen = 2
 # If an explicit setting is given, it will be used for all parts of the codebase.
 #new-symbol-mangling = true|false (see comment)
 
-# Select LTO mode that will be used for compiling rustc. By default, thin local LTO (LTO within a
-# single crate) is used. You can also select "thin" or "fat" to apply Thin/Fat LTO on the
-# `rustc_driver` dylib.
-#lto = thin-local
+# Select LTO mode that will be used for compiling rustc. By default, thin local LTO
+# (LTO within a single crate) is used (like for any Rust crate). You can also select
+# "thin" or "fat" to apply Thin/Fat LTO to the `rustc_driver` dylib.
+#lto = "thin-local"
 
 # =============================================================================
 # Options for specific targets

Diff for: src/bootstrap/compile.rs

@@ -701,6 +701,28 @@ impl Step for Rustc {
             ));
         }
 
+        // cfg(bootstrap): remove if condition once the bootstrap compiler supports dylib LTO
+        if compiler.stage != 0 {
+            match builder.config.rust_lto {
+                RustcLto::Thin | RustcLto::Fat => {
+                    // Since using LTO for optimizing dylibs is currently experimental,
+                    // we need to pass -Zdylib-lto.
+                    cargo.rustflag("-Zdylib-lto");
+                    // Cargo by default passes `-Cembed-bitcode=no` and doesn't pass `-Clto` when
+                    // compiling dylibs (and their dependencies), even when LTO is enabled for the
+                    // crate. Therefore, we need to override `-Clto` and `-Cembed-bitcode` here.
+                    let lto_type = match builder.config.rust_lto {
+                        RustcLto::Thin => "thin",
+                        RustcLto::Fat => "fat",
+                        _ => unreachable!(),
+                    };
+                    cargo.rustflag(&format!("-Clto={}", lto_type));
+                    cargo.rustflag("-Cembed-bitcode=yes");
+                }
+                RustcLto::ThinLocal => { /* Do nothing, this is the default */ }
+            }
+        }
+
         builder.info(&format!(
             "Building stage{} compiler artifacts ({} -> {})",
             compiler.stage, &compiler.host, target

Diff for: src/bootstrap/config.rs

@@ -321,12 +321,12 @@ impl SplitDebuginfo {
 }
 
 /// LTO mode used for compiling rustc itself.
-#[derive(Default)]
+#[derive(Default, Clone)]
 pub enum RustcLto {
     #[default]
     ThinLocal,
     Thin,
-    Fat
+    Fat,
 }
 
 impl std::str::FromStr for RustcLto {
@@ -1201,8 +1201,7 @@ impl Config {
             config.rust_lto = rust
                 .lto
                 .as_deref()
-                .map(RustcLto::from_str)
-                .map(|v| v.expect("invalid value for rust.lto"))
+                .map(|value| RustcLto::from_str(value).unwrap())
                 .unwrap_or_default();
         } else {
             config.rust_profile_use = flags.rust_profile_use;

Diff for: src/doc/unstable-book/src/compiler-flags/dylib-lto.md

@@ -0,0 +1,4 @@
+## `dylib-lto`
+
+This option enables using LTO for the `dylib` crate type. This is currently only used for compiling
+`rustc` itself (more specifically, the `librustc_driver` dylib).

Diff for: src/test/rustdoc-ui/z-help.stdout

@@ -36,6 +36,7 @@
     -Z                       dump-mir-graphviz=val -- in addition to `.mir` files, create graphviz `.dot` files (and with `-Z instrument-coverage`, also create a `.dot` file for the MIR-derived coverage graph) (default: no)
     -Z                       dump-mir-spanview=val -- in addition to `.mir` files, create `.html` files to view spans for all `statement`s (including terminators), only `terminator` spans, or computed `block` spans (one span encompassing a block's terminator and all statements). If `-Z instrument-coverage` is also enabled, create an additional `.html` file showing the computed coverage spans.
     -Z                           dwarf-version=val -- version of DWARF debug information to emit (default: 2 or 4, depending on platform)
+    -Z                               dylib-lto=val -- enables LTO for dylib crate type
     -Z                        emit-stack-sizes=val -- emit a section containing stack size metadata (default: no)
     -Z                           emit-thin-lto=val -- emit the bc module with thin LTO info (default: yes)
     -Z               export-executable-symbols=val -- export symbols from executables, as if they were dynamic libraries