Auto merge of #133092 - madsmtm:bootstrap-deployment-target, r=Mark-Simulacrum,jieyouxu

Always set the deployment target when building std

`cc` has [a bug/feature](rust-lang/cc-rs#1171) (I guess depending on how you look at it) where the default deployment target is taken from the SDK instead of from `rustc`. This causes `compiler-builtins` to build `compiler-rt` with the wrong deployment target on iOS.

I've been meaning to change how `cc` works in this regard, but that's a lengthy process, so let's fix it in bootstrap for now.

The behaviour can be seen locally with `./x build library --set build.optimized-compiler-builtins=true` for various target triples, and then inspecting with `otool -l build/host/stage1/lib/rustlib/*/lib/libcompiler_builtins-*.rlib | rg 'minos|version'`. I have added a rmake test that ensures that we now have the same version everywhere.

Fixes #128419
Fixes rust-lang/compiler-builtins#650
Fixes #136523
See also rust-lang/cargo#13115, rust-lang/cc-rs#1171, #136113
See #133092 (comment) for a description of how the change works.

try-job: i686-gnu-1
try-job: i686-gnu-2
try-job: x86_64-apple-1
try-job: aarch64-apple
try-job: dist-apple-various
try-job: dist-aarch64-apple
try-job: dist-various-2
try-job: x86_64-fuchsia

Author: bors

Diff for: src/bootstrap/src/core/build_steps/compile.rs

@@ -443,8 +443,49 @@ fn compiler_rt_for_profiler(builder: &Builder<'_>) -> PathBuf {
 /// Configure cargo to compile the standard library, adding appropriate env vars
 /// and such.
 pub fn std_cargo(builder: &Builder<'_>, target: TargetSelection, stage: u32, cargo: &mut Cargo) {
-    if let Some(target) = env::var_os("MACOSX_STD_DEPLOYMENT_TARGET") {
-        cargo.env("MACOSX_DEPLOYMENT_TARGET", target);
+    // rustc already ensures that it builds with the minimum deployment
+    // target, so ideally we shouldn't need to do anything here.
+    //
+    // However, `cc` currently defaults to a higher version for backwards
+    // compatibility, which means that compiler-rt, which is built via
+    // compiler-builtins' build script, gets built with a higher deployment
+    // target. This in turn causes warnings while linking, and is generally
+    // a compatibility hazard.
+    //
+    // So, at least until https://github.com/rust-lang/cc-rs/issues/1171, or
+    // perhaps https://github.com/rust-lang/cargo/issues/13115 is resolved, we
+    // explicitly set the deployment target environment variables to avoid
+    // this issue.
+    //
+    // This place also serves as an extension point if we ever wanted to raise
+    // rustc's default deployment target while keeping the prebuilt `std` at
+    // a lower version, so it's kinda nice to have in any case.
+    if target.contains("apple") && !builder.config.dry_run() {
+        // Query rustc for the deployment target, and the associated env var.
+        // The env var is one of the standard `*_DEPLOYMENT_TARGET` vars, i.e.
+        // `MACOSX_DEPLOYMENT_TARGET`, `IPHONEOS_DEPLOYMENT_TARGET`, etc.
+        let mut cmd = command(builder.rustc(cargo.compiler()));
+        cmd.arg("--target").arg(target.rustc_target_arg());
+        cmd.arg("--print=deployment-target");
+        let output = cmd.run_capture_stdout(builder).stdout();
+
+        let (env_var, value) = output.split_once('=').unwrap();
+        // Unconditionally set the env var (if it was set in the environment
+        // already, rustc should've picked that up).
+        cargo.env(env_var.trim(), value.trim());
+
+        // Allow CI to override the deployment target for `std` on macOS.
+        //
+        // This is useful because we might want the host tooling LLVM, `rustc`
+        // and Cargo to have a different deployment target than `std` itself
+        // (currently, these two versions are the same, but in the past, we
+        // supported macOS 10.7 for user code and macOS 10.8 in host tooling).
+        //
+        // It is not necessary on the other platforms, since only macOS has
+        // support for host tooling.
+        if let Some(target) = env::var_os("MACOSX_STD_DEPLOYMENT_TARGET") {
+            cargo.env("MACOSX_DEPLOYMENT_TARGET", target);
+        }
     }
 
     // Paths needed by `library/profiler_builtins/build.rs`.
@@ -1547,7 +1588,8 @@ pub fn compiler_file(
         return PathBuf::new();
     }
     let mut cmd = command(compiler);
-    cmd.args(builder.cflags(target, GitRepo::Rustc, c));
+    cmd.args(builder.cc_handled_clags(target, c));
+    cmd.args(builder.cc_unhandled_cflags(target, GitRepo::Rustc, c));
     cmd.arg(format!("-print-file-name={file}"));
     let out = cmd.run_capture_stdout(builder).stdout();
     PathBuf::from(out.trim())

Diff for: src/bootstrap/src/core/build_steps/llvm.rs

@@ -779,9 +779,15 @@ fn configure_cmake(
     }
 
     cfg.build_arg("-j").build_arg(builder.jobs().to_string());
+    // FIXME(madsmtm): Allow `cmake-rs` to select flags by itself by passing
+    // our flags via `.cflag`/`.cxxflag` instead.
+    //
+    // Needs `suppressed_compiler_flag_prefixes` to be gone, and hence
+    // https://github.com/llvm/llvm-project/issues/88780 to be fixed.
     let mut cflags: OsString = builder
-        .cflags(target, GitRepo::Llvm, CLang::C)
+        .cc_handled_clags(target, CLang::C)
         .into_iter()
+        .chain(builder.cc_unhandled_cflags(target, GitRepo::Llvm, CLang::C))
         .filter(|flag| {
             !suppressed_compiler_flag_prefixes
                 .iter()
@@ -800,8 +806,9 @@ fn configure_cmake(
     }
     cfg.define("CMAKE_C_FLAGS", cflags);
     let mut cxxflags: OsString = builder
-        .cflags(target, GitRepo::Llvm, CLang::Cxx)
+        .cc_handled_clags(target, CLang::Cxx)
         .into_iter()
+        .chain(builder.cc_unhandled_cflags(target, GitRepo::Llvm, CLang::Cxx))
         .filter(|flag| {
             !suppressed_compiler_flag_prefixes
                 .iter()

Diff for: src/bootstrap/src/core/build_steps/test.rs

@@ -2011,14 +2011,18 @@ NOTE: if you're sure you want to do this, please open an issue as to why. In the
         // Only pass correct values for these flags for the `run-make` suite as it
         // requires that a C++ compiler was configured which isn't always the case.
         if !builder.config.dry_run() && mode == "run-make" {
+            let mut cflags = builder.cc_handled_clags(target, CLang::C);
+            cflags.extend(builder.cc_unhandled_cflags(target, GitRepo::Rustc, CLang::C));
+            let mut cxxflags = builder.cc_handled_clags(target, CLang::Cxx);
+            cxxflags.extend(builder.cc_unhandled_cflags(target, GitRepo::Rustc, CLang::Cxx));
             cmd.arg("--cc")
                 .arg(builder.cc(target))
                 .arg("--cxx")
                 .arg(builder.cxx(target).unwrap())
                 .arg("--cflags")
-                .arg(builder.cflags(target, GitRepo::Rustc, CLang::C).join(" "))
+                .arg(cflags.join(" "))
                 .arg("--cxxflags")
-                .arg(builder.cflags(target, GitRepo::Rustc, CLang::Cxx).join(" "));
+                .arg(cxxflags.join(" "));
             copts_passed = true;
             if let Some(ar) = builder.ar(target) {
                 cmd.arg("--ar").arg(ar);

Diff for: src/bootstrap/src/core/builder/cargo.rs

@@ -323,8 +323,15 @@ impl Cargo {
             let cc = ccacheify(&builder.cc(target));
             self.command.env(format!("CC_{triple_underscored}"), &cc);
 
-            let cflags = builder.cflags(target, GitRepo::Rustc, CLang::C).join(" ");
-            self.command.env(format!("CFLAGS_{triple_underscored}"), &cflags);
+            // Extend `CXXFLAGS_$TARGET` with our extra flags.
+            let env = format!("CFLAGS_{triple_underscored}");
+            let mut cflags =
+                builder.cc_unhandled_cflags(target, GitRepo::Rustc, CLang::C).join(" ");
+            if let Ok(var) = std::env::var(&env) {
+                cflags.push(' ');
+                cflags.push_str(&var);
+            }
+            self.command.env(env, &cflags);
 
             if let Some(ar) = builder.ar(target) {
                 let ranlib = format!("{} s", ar.display());
@@ -335,10 +342,17 @@ impl Cargo {
 
             if let Ok(cxx) = builder.cxx(target) {
                 let cxx = ccacheify(&cxx);
-                let cxxflags = builder.cflags(target, GitRepo::Rustc, CLang::Cxx).join(" ");
-                self.command
-                    .env(format!("CXX_{triple_underscored}"), &cxx)
-                    .env(format!("CXXFLAGS_{triple_underscored}"), cxxflags);
+                self.command.env(format!("CXX_{triple_underscored}"), &cxx);
+
+                // Extend `CXXFLAGS_$TARGET` with our extra flags.
+                let env = format!("CXXFLAGS_{triple_underscored}");
+                let mut cxxflags =
+                    builder.cc_unhandled_cflags(target, GitRepo::Rustc, CLang::Cxx).join(" ");
+                if let Ok(var) = std::env::var(&env) {
+                    cxxflags.push(' ');
+                    cxxflags.push_str(&var);
+                }
+                self.command.env(&env, cxxflags);
             }
         }
 

Diff for: src/bootstrap/src/lib.rs

@@ -249,6 +249,7 @@ impl Mode {
     }
 }
 
+#[derive(Debug, Hash, Clone, Copy, PartialEq, Eq, PartialOrd, Ord)]
 pub enum CLang {
     C,
     Cxx,
@@ -1178,9 +1179,9 @@ Executed at: {executed_at}"#,
         self.cc.borrow()[&target].path().into()
     }
 
-    /// Returns a list of flags to pass to the C compiler for the target
-    /// specified.
-    fn cflags(&self, target: TargetSelection, which: GitRepo, c: CLang) -> Vec<String> {
+    /// Returns C flags that `cc-rs` thinks should be enabled for the
+    /// specified target by default.
+    fn cc_handled_clags(&self, target: TargetSelection, c: CLang) -> Vec<String> {
         if self.config.dry_run() {
             return Vec::new();
         }
@@ -1189,14 +1190,23 @@ Executed at: {executed_at}"#,
             CLang::Cxx => self.cxx.borrow()[&target].clone(),
         };
 
-        // Filter out -O and /O (the optimization flags) that we picked up from
-        // cc-rs because the build scripts will determine that for themselves.
-        let mut base = base
-            .args()
+        // Filter out -O and /O (the optimization flags) that we picked up
+        // from cc-rs, that's up to the caller to figure out.
+        base.args()
             .iter()
             .map(|s| s.to_string_lossy().into_owned())
             .filter(|s| !s.starts_with("-O") && !s.starts_with("/O"))
-            .collect::<Vec<String>>();
+            .collect::<Vec<String>>()
+    }
+
+    /// Returns extra C flags that `cc-rs` doesn't handle.
+    fn cc_unhandled_cflags(
+        &self,
+        target: TargetSelection,
+        which: GitRepo,
+        c: CLang,
+    ) -> Vec<String> {
+        let mut base = Vec::new();
 
         // If we're compiling C++ on macOS then we add a flag indicating that
         // we want libc++ (more filled out than libstdc++), ensuring that

Diff for: src/bootstrap/src/utils/cc_detect.rs

@@ -142,7 +142,8 @@ pub fn find_target(build: &Build, target: TargetSelection) {
     };
 
     build.cc.borrow_mut().insert(target, compiler.clone());
-    let cflags = build.cflags(target, GitRepo::Rustc, CLang::C);
+    let mut cflags = build.cc_handled_clags(target, CLang::C);
+    cflags.extend(build.cc_unhandled_cflags(target, GitRepo::Rustc, CLang::C));
 
     // If we use llvm-libunwind, we will need a C++ compiler as well for all targets
     // We'll need one anyways if the target triple is also a host triple
@@ -168,7 +169,8 @@ pub fn find_target(build: &Build, target: TargetSelection) {
     build.verbose(|| println!("CC_{} = {:?}", target.triple, build.cc(target)));
     build.verbose(|| println!("CFLAGS_{} = {cflags:?}", target.triple));
     if let Ok(cxx) = build.cxx(target) {
-        let cxxflags = build.cflags(target, GitRepo::Rustc, CLang::Cxx);
+        let mut cxxflags = build.cc_handled_clags(target, CLang::Cxx);
+        cxxflags.extend(build.cc_unhandled_cflags(target, GitRepo::Rustc, CLang::Cxx));
         build.verbose(|| println!("CXX_{} = {cxx:?}", target.triple));
         build.verbose(|| println!("CXXFLAGS_{} = {cxxflags:?}", target.triple));
     }

Diff for: src/ci/github-actions/jobs.yml

@@ -59,6 +59,7 @@ envs:
     SCRIPT: ./x.py --stage 2 test --skip tests/ui --skip tests/rustdoc -- --exact
     RUST_CONFIGURE_ARGS: --build=x86_64-apple-darwin --enable-sanitizers --enable-profiler --set rust.jemalloc
     RUSTC_RETRY_LINKER_ON_SEGFAULT: 1
+    # Ensure that host tooling is tested on our minimum supported macOS version.
     MACOSX_DEPLOYMENT_TARGET: 10.12
     MACOSX_STD_DEPLOYMENT_TARGET: 10.12
     SELECT_XCODE: /Applications/Xcode_15.2.app
@@ -370,7 +371,9 @@ auto:
       SCRIPT: ./x.py dist bootstrap --include-default-paths --host=x86_64-apple-darwin --target=x86_64-apple-darwin
       RUST_CONFIGURE_ARGS: --enable-full-tools --enable-sanitizers --enable-profiler --set rust.jemalloc --set rust.lto=thin --set rust.codegen-units=1
       RUSTC_RETRY_LINKER_ON_SEGFAULT: 1
+      # Ensure that host tooling is built to support our minimum support macOS version.
       MACOSX_DEPLOYMENT_TARGET: 10.12
+      MACOSX_STD_DEPLOYMENT_TARGET: 10.12
       SELECT_XCODE: /Applications/Xcode_15.2.app
       NO_LLVM_ASSERTIONS: 1
       NO_DEBUG_ASSERTIONS: 1
@@ -386,7 +389,10 @@ auto:
       # https://github.com/rust-lang/rust/issues/129069
       RUST_CONFIGURE_ARGS: --enable-sanitizers --enable-profiler --set rust.jemalloc --set target.aarch64-apple-ios-macabi.sanitizers=false --set target.x86_64-apple-ios-macabi.sanitizers=false
       RUSTC_RETRY_LINKER_ON_SEGFAULT: 1
+      # Ensure that host tooling is built to support our minimum support macOS version.
+      # FIXME(madsmtm): This might be redundant, as we're not building host tooling here (?)
       MACOSX_DEPLOYMENT_TARGET: 10.12
+      MACOSX_STD_DEPLOYMENT_TARGET: 10.12
       SELECT_XCODE: /Applications/Xcode_15.2.app
       NO_LLVM_ASSERTIONS: 1
       NO_DEBUG_ASSERTIONS: 1
@@ -404,7 +410,6 @@ auto:
       <<: *env-x86_64-apple-tests
     <<: *job-macos-xl
 
-  # This target only needs to support 11.0 and up as nothing else supports the hardware
   - name: dist-aarch64-apple
     env:
       SCRIPT: ./x.py dist bootstrap --include-default-paths --host=aarch64-apple-darwin --target=aarch64-apple-darwin
@@ -419,6 +424,8 @@ auto:
       RUSTC_RETRY_LINKER_ON_SEGFAULT: 1
       SELECT_XCODE: /Applications/Xcode_15.4.app
       USE_XCODE_CLANG: 1
+      # Aarch64 tooling only needs to support macOS 11.0 and up as nothing else
+      # supports the hardware.
       MACOSX_DEPLOYMENT_TARGET: 11.0
       MACOSX_STD_DEPLOYMENT_TARGET: 11.0
       NO_LLVM_ASSERTIONS: 1
@@ -428,7 +435,6 @@ auto:
       CODEGEN_BACKENDS: llvm,cranelift
     <<: *job-macos-m1
 
-  # This target only needs to support 11.0 and up as nothing else supports the hardware
   - name: aarch64-apple
     env:
       SCRIPT: ./x.py --stage 2 test --host=aarch64-apple-darwin --target=aarch64-apple-darwin
@@ -439,6 +445,8 @@ auto:
       RUSTC_RETRY_LINKER_ON_SEGFAULT: 1
       SELECT_XCODE: /Applications/Xcode_15.4.app
       USE_XCODE_CLANG: 1
+      # Aarch64 tooling only needs to support macOS 11.0 and up as nothing else
+      # supports the hardware, so only need to test it there.
       MACOSX_DEPLOYMENT_TARGET: 11.0
       MACOSX_STD_DEPLOYMENT_TARGET: 11.0
       NO_LLVM_ASSERTIONS: 1

Diff for: tests/run-make/apple-deployment-target/rmake.rs

@@ -7,7 +7,11 @@
 
 //@ only-apple
 
-use run_make_support::{apple_os, cmd, run_in_tmpdir, rustc, target};
+use std::collections::HashSet;
+
+use run_make_support::{
+    apple_os, cmd, has_extension, path, regex, run_in_tmpdir, rustc, shallow_find_files, target,
+};
 
 /// Run vtool to check the `minos` field in LC_BUILD_VERSION.
 ///
@@ -166,4 +170,21 @@ fn main() {
         rustc().env_remove(env_var).run();
         minos("foo.o", default_version);
     });
+
+    // Test that all binaries in rlibs produced by `rustc` have the same version.
+    // Regression test for https://github.com/rust-lang/rust/issues/128419.
+    let sysroot = rustc().print("sysroot").run().stdout_utf8();
+    let target_sysroot = path(sysroot.trim()).join("lib/rustlib").join(target()).join("lib");
+    let rlibs = shallow_find_files(&target_sysroot, |path| has_extension(path, "rlib"));
+
+    let output = cmd("otool").arg("-l").args(rlibs).run().stdout_utf8();
+    let re = regex::Regex::new(r"(minos|version) ([0-9.]*)").unwrap();
+    let mut versions = HashSet::new();
+    for (_, [_, version]) in re.captures_iter(&output).map(|c| c.extract()) {
+        versions.insert(version);
+    }
+    // FIXME(madsmtm): See above for aarch64-apple-watchos.
+    if versions.len() != 1 && target() != "aarch64-apple-watchos" {
+        panic!("std rlibs contained multiple different deployment target versions: {versions:?}");
+    }
 }