Emit better error messages when invoking xcrun

Also allow the SDK path to be non-UTF-8.

Author: madsmtm

Diff for: compiler/rustc_codegen_ssa/messages.ftl

@@ -10,8 +10,6 @@ codegen_ssa_apple_deployment_target_invalid =
 codegen_ssa_apple_deployment_target_too_low =
     deployment target in {$env_var} was set to {$version}, but the minimum supported by `rustc` is {$os_min}
 
-codegen_ssa_apple_sdk_error_sdk_path = failed to get {$sdk_name} SDK path: {$error}
-
 codegen_ssa_archive_build_failure = failed to build archive at `{$path}`: {$error}
 
 codegen_ssa_atomic_compare_exchange = Atomic compare-exchange intrinsic missing failure memory ordering
@@ -400,3 +398,20 @@ codegen_ssa_use_cargo_directive = use the `cargo:rustc-link-lib` directive to sp
 codegen_ssa_version_script_write_failure = failed to write version script: {$error}
 
 codegen_ssa_visual_studio_not_installed = you may need to install Visual Studio build tools with the "C++ build tools" workload
+
+codegen_ssa_xcrun_command_line_tools_insufficient =
+    when compiling for iOS, tvOS, visionOS or watchOS, you need a full installation of Xcode
+
+codegen_ssa_xcrun_failed_invoking = invoking `{$command_formatted}` to find {$sdk_name}.sdk failed: {$error}
+
+codegen_ssa_xcrun_found_developer_dir = found active developer directory at "{$developer_dir}"
+
+# `xcrun` already outputs a message about missing Xcode installation, so we only augment it with details about env vars.
+codegen_ssa_xcrun_no_developer_dir =
+    pass the path of an Xcode installation via the DEVELOPER_DIR environment variable, or an SDK with the SDKROOT environment variable
+
+codegen_ssa_xcrun_sdk_path_warning = output of `xcrun` while finding {$sdk_name}.sdk
+    .note = {$stderr}
+
+codegen_ssa_xcrun_unsuccessful = failed running `{$command_formatted}` to find {$sdk_name}.sdk
+    .note = {$stdout}{$stderr}

Diff for: compiler/rustc_codegen_ssa/src/back/apple.rs

@@ -1,12 +1,18 @@
 use std::env;
+use std::ffi::OsString;
 use std::fmt::{Display, from_fn};
 use std::num::ParseIntError;
+use std::path::PathBuf;
+use std::process::Command;
 
+use itertools::Itertools;
 use rustc_middle::middle::exported_symbols::SymbolExportKind;
 use rustc_session::Session;
 use rustc_target::spec::Target;
+use tracing::debug;
 
-use crate::errors::AppleDeploymentTarget;
+use crate::errors::{AppleDeploymentTarget, XcrunError, XcrunSdkPathWarning};
+use crate::fluent_generated as fluent;
 
 #[cfg(test)]
 mod tests;
@@ -271,3 +277,131 @@ pub(super) fn add_version_to_llvm_target(
         format!("{arch}-{vendor}-{os}{major}.{minor}.{patch}")
     }
 }
+
+pub(super) fn get_sdk_root(sess: &Session) -> Option<PathBuf> {
+    let sdk_name = sdk_name(&sess.target);
+
+    match xcrun_show_sdk_path(sdk_name, sess.verbose_internals()) {
+        Ok((path, stderr)) => {
+            // Emit extra stderr, such as if `-verbose` was passed, or if `xcrun` emitted a warning.
+            if !stderr.is_empty() {
+                sess.dcx().emit_warn(XcrunSdkPathWarning { sdk_name, stderr });
+            }
+            Some(path)
+        }
+        Err(err) => {
+            let mut diag = sess.dcx().create_err(err);
+
+            // Recognize common error cases, and give more Rust-specific error messages for those.
+            if let Some(developer_dir) = xcode_select_developer_dir() {
+                diag.arg("developer_dir", &developer_dir);
+                diag.note(fluent::codegen_ssa_xcrun_found_developer_dir);
+                if developer_dir.as_os_str().to_string_lossy().contains("CommandLineTools") {
+                    if sdk_name != "MacOSX" {
+                        diag.help(fluent::codegen_ssa_xcrun_command_line_tools_insufficient);
+                    }
+                }
+            } else {
+                diag.help(fluent::codegen_ssa_xcrun_no_developer_dir);
+            }
+
+            diag.emit();
+            None
+        }
+    }
+}
+
+/// Invoke `xcrun --sdk $sdk_name --show-sdk-path` to get the SDK path.
+///
+/// The exact logic that `xcrun` uses is unspecified (see `man xcrun` for a few details), and may
+/// change between macOS and Xcode versions, but it roughly boils down to finding the active
+/// developer directory, and then invoking `xcodebuild -sdk $sdk_name -version` to get the SDK
+/// details.
+///
+/// Finding the developer directory is roughly done by looking at, in order:
+/// - The `DEVELOPER_DIR` environment variable.
+/// - The `/var/db/xcode_select_link` symlink (set by `xcode-select --switch`).
+/// - `/Applications/Xcode.app` (hardcoded fallback path).
+/// - `/Library/Developer/CommandLineTools` (hardcoded fallback path).
+///
+/// Note that `xcrun` caches its result, but with a cold cache this whole operation can be quite
+/// slow, especially so the first time it's run after a reboot.
+fn xcrun_show_sdk_path(
+    sdk_name: &'static str,
+    verbose: bool,
+) -> Result<(PathBuf, String), XcrunError> {
+    let mut cmd = Command::new("xcrun");
+    if verbose {
+        cmd.arg("--verbose");
+    }
+    // The `--sdk` parameter is the same as in xcodebuild, namely either an absolute path to an SDK,
+    // or the (lowercase) canonical name of an SDK.
+    cmd.arg("--sdk");
+    cmd.arg(&sdk_name.to_lowercase());
+    cmd.arg("--show-sdk-path");
+
+    // We do not stream stdout/stderr lines directly to the user, since whether they are warnings or
+    // errors depends on the status code at the end.
+    let output = cmd.output().map_err(|error| XcrunError::FailedInvoking {
+        sdk_name,
+        command_formatted: format!("{cmd:?}"),
+        error,
+    })?;
+
+    // It is fine to do lossy conversion here, non-UTF-8 paths are quite rare on macOS nowadays
+    // (only possible with the HFS+ file system), and we only use it for error messages.
+    let stderr = String::from_utf8_lossy_owned(output.stderr);
+    if !stderr.is_empty() {
+        debug!(stderr, "original xcrun stderr");
+    }
+
+    // Some versions of `xcodebuild` output beefy errors when invoked via `xcrun`,
+    // but these are usually red herrings.
+    let stderr = stderr
+        .lines()
+        .filter(|line| {
+            !line.contains("Writing error result bundle")
+                && !line.contains("Requested but did not find extension point with identifier")
+        })
+        .join("\n");
+
+    if output.status.success() {
+        Ok((stdout_to_path(output.stdout), stderr))
+    } else {
+        // Output both stdout and stderr, since shims of `xcrun` (such as the one provided by
+        // nixpkgs), do not always use stderr for errors.
+        let stdout = String::from_utf8_lossy_owned(output.stdout).trim().to_string();
+        Err(XcrunError::Unsuccessful {
+            sdk_name,
+            command_formatted: format!("{cmd:?}"),
+            stdout,
+            stderr,
+        })
+    }
+}
+
+/// Invoke `xcode-select --print-path`, and return the current developer directory.
+///
+/// NOTE: We don't do any error handling here, this is only used as a canary in diagnostics (`xcrun`
+/// will have already emitted the relevant error information).
+fn xcode_select_developer_dir() -> Option<PathBuf> {
+    let mut cmd = Command::new("xcode-select");
+    cmd.arg("--print-path");
+    let output = cmd.output().ok()?;
+    if !output.status.success() {
+        return None;
+    }
+    Some(stdout_to_path(output.stdout))
+}
+
+fn stdout_to_path(mut stdout: Vec<u8>) -> PathBuf {
+    // Remove trailing newline.
+    if let Some(b'\n') = stdout.last() {
+        let _ = stdout.pop().unwrap();
+    }
+    #[cfg(unix)]
+    let path = <OsString as std::os::unix::ffi::OsStringExt>::from_vec(stdout);
+    #[cfg(not(unix))] // Unimportant, this is only used on macOS
+    let path = OsString::from(String::from_utf8(stdout).unwrap());
+    PathBuf::from(path)
+}

Diff for: compiler/rustc_codegen_ssa/src/back/apple/tests.rs

@@ -1,4 +1,4 @@
-use super::{add_version_to_llvm_target, parse_version};
+use super::*;
 
 #[test]
 fn test_add_version_to_llvm_target() {
@@ -19,3 +19,69 @@ fn test_parse_version() {
     assert_eq!(parse_version("10.12.6"), Ok((10, 12, 6)));
     assert_eq!(parse_version("9999.99.99"), Ok((9999, 99, 99)));
 }
+
+#[test]
+#[cfg_attr(not(target_os = "macos"), ignore = "xcode-select is only available on macOS")]
+fn lookup_developer_dir() {
+    let _developer_dir = xcode_select_developer_dir().unwrap();
+}
+
+#[test]
+#[cfg_attr(not(target_os = "macos"), ignore = "xcrun is only available on macOS")]
+fn lookup_sdk() {
+    let (sdk_path, stderr) = xcrun_show_sdk_path("MacOSX", false).unwrap();
+    // Check that the found SDK is valid.
+    assert!(sdk_path.join("SDKSettings.plist").exists());
+    assert_eq!(stderr, "");
+
+    // Test that the SDK root is a subdir of the developer directory.
+    if let Some(developer_dir) = xcode_select_developer_dir() {
+        // Only run this test if SDKROOT is not set (otherwise xcrun may look up via. that).
+        if std::env::var_os("SDKROOT").is_some() {
+            assert!(sdk_path.starts_with(&developer_dir));
+        }
+    }
+}
+
+#[test]
+#[cfg_attr(not(target_os = "macos"), ignore = "xcrun is only available on macOS")]
+fn lookup_sdk_verbose() {
+    let (_, stderr) = xcrun_show_sdk_path("MacOSX", true).unwrap();
+    // Newer xcrun versions should emit something like this:
+    //
+    //     xcrun: note: looking up SDK with 'xcodebuild -sdk macosx -version Path'
+    //     xcrun: note: xcrun_db = '/var/.../xcrun_db'
+    //     xcrun: note: lookup resolved to: '...'
+    //     xcrun: note: database key is: ...
+    //
+    // Or if the value is already cached, something like this:
+    //
+    //     xcrun: note: database key is: ...
+    //     xcrun: note: lookup resolved in '/var/.../xcrun_db' : '...'
+    assert!(
+        stderr.contains("xcrun: note: lookup resolved"),
+        "stderr should contain lookup note: {stderr}",
+    );
+}
+
+#[test]
+#[cfg_attr(not(target_os = "macos"), ignore = "xcrun is only available on macOS")]
+fn try_lookup_invalid_sdk() {
+    // As a proxy for testing all the different ways that `xcrun` can fail,
+    // test the case where an SDK was not found.
+    let err = xcrun_show_sdk_path("invalid", false).unwrap_err();
+    let XcrunError::Unsuccessful { stderr, .. } = err else {
+        panic!("unexpected error kind: {err:?}");
+    };
+    // Either one of (depending on if using Command Line Tools or full Xcode):
+    // xcrun: error: SDK "invalid" cannot be located
+    // xcodebuild: error: SDK "invalid" cannot be located.
+    assert!(
+        stderr.contains(r#"error: SDK "invalid" cannot be located"#),
+        "stderr should contain xcodebuild note: {stderr}",
+    );
+    assert!(
+        stderr.contains("xcrun: error: unable to lookup item 'Path' in SDK 'invalid'"),
+        "stderr should contain xcrun note: {stderr}",
+    );
+}

Diff for: compiler/rustc_codegen_ssa/src/back/link.rs

@@ -3213,15 +3213,7 @@ fn add_apple_sdk(cmd: &mut dyn Linker, sess: &Session, flavor: LinkerFlavor) ->
         return None;
     }
 
-    let sdk_name = apple::sdk_name(&sess.target).to_lowercase();
-
-    let sdk_root = match sess.time("get_apple_sdk_root", || get_apple_sdk_root(&sdk_name)) {
-        Ok(s) => s,
-        Err(e) => {
-            sess.dcx().emit_err(e);
-            return None;
-        }
-    };
+    let sdk_root = sess.time("get_apple_sdk_root", || get_apple_sdk_root(sess))?;
 
     match flavor {
         LinkerFlavor::Darwin(Cc::Yes, _) => {
@@ -3231,28 +3223,32 @@ fn add_apple_sdk(cmd: &mut dyn Linker, sess: &Session, flavor: LinkerFlavor) ->
             // This is admittedly a bit strange, as on most targets
             // `-isysroot` only applies to include header files, but on Apple
             // targets this also applies to libraries and frameworks.
-            cmd.cc_args(&["-isysroot", &sdk_root]);
+            cmd.cc_arg("-isysroot");
+            cmd.cc_arg(&sdk_root);
         }
         LinkerFlavor::Darwin(Cc::No, _) => {
-            cmd.link_args(&["-syslibroot", &sdk_root]);
+            cmd.link_arg("-syslibroot");
+            cmd.link_arg(&sdk_root);
         }
         _ => unreachable!(),
     }
 
-    Some(sdk_root.into())
+    Some(sdk_root)
 }
 
-fn get_apple_sdk_root(sdk_name: &str) -> Result<String, errors::AppleSdkRootError<'_>> {
-    // Following what clang does
-    // (https://github.com/llvm/llvm-project/blob/
-    // 296a80102a9b72c3eda80558fb78a3ed8849b341/clang/lib/Driver/ToolChains/Darwin.cpp#L1661-L1678)
-    // to allow the SDK path to be set. (For clang, xcrun sets
-    // SDKROOT; for rustc, the user or build system can set it, or we
-    // can fall back to checking for xcrun on PATH.)
+fn get_apple_sdk_root(sess: &Session) -> Option<PathBuf> {
     if let Ok(sdkroot) = env::var("SDKROOT") {
-        let p = Path::new(&sdkroot);
-        match sdk_name {
-            // Ignore `SDKROOT` if it's clearly set for the wrong platform.
+        let p = PathBuf::from(&sdkroot);
+
+        // Ignore invalid SDKs, similar to what clang does:
+        // https://github.com/llvm/llvm-project/blob/llvmorg-19.1.6/clang/lib/Driver/ToolChains/Darwin.cpp#L2212-L2229
+        //
+        // NOTE: Things are complicated here by the fact that `rustc` can be run by Cargo to compile
+        // build scripts and proc-macros for the host, and thus we need to ignore SDKROOT if it's
+        // clearly set for the wrong platform.
+        //
+        // FIXME(madsmtm): Make this more robust (maybe read `SDKSettings.json` like Clang does?).
+        match &*apple::sdk_name(&sess.target).to_lowercase() {
             "appletvos"
                 if sdkroot.contains("TVSimulator.platform")
                     || sdkroot.contains("MacOSX.platform") => {}
@@ -3279,26 +3275,11 @@ fn get_apple_sdk_root(sdk_name: &str) -> Result<String, errors::AppleSdkRootErro
                 if sdkroot.contains("XROS.platform") || sdkroot.contains("MacOSX.platform") => {}
             // Ignore `SDKROOT` if it's not a valid path.
             _ if !p.is_absolute() || p == Path::new("/") || !p.exists() => {}
-            _ => return Ok(sdkroot),
+            _ => return Some(p),
         }
     }
-    let res =
-        Command::new("xcrun").arg("--show-sdk-path").arg("-sdk").arg(sdk_name).output().and_then(
-            |output| {
-                if output.status.success() {
-                    Ok(String::from_utf8(output.stdout).unwrap())
-                } else {
-                    let error = String::from_utf8(output.stderr);
-                    let error = format!("process exit with error: {}", error.unwrap());
-                    Err(io::Error::new(io::ErrorKind::Other, &error[..]))
-                }
-            },
-        );
 
-    match res {
-        Ok(output) => Ok(output.trim().to_string()),
-        Err(error) => Err(errors::AppleSdkRootError::SdkPath { sdk_name, error }),
-    }
+    apple::get_sdk_root(sess)
 }
 
 /// When using the linker flavors opting in to `lld`, add the necessary paths and arguments to

Diff for: compiler/rustc_codegen_ssa/src/errors.rs

@@ -746,12 +746,6 @@ pub(crate) enum AppleDeploymentTarget {
     TooLow { env_var: &'static str, version: String, os_min: String },
 }
 
-#[derive(Diagnostic)]
-pub(crate) enum AppleSdkRootError<'a> {
-    #[diag(codegen_ssa_apple_sdk_error_sdk_path)]
-    SdkPath { sdk_name: &'a str, error: Error },
-}
-
 #[derive(Diagnostic)]
 #[diag(codegen_ssa_read_file)]
 pub(crate) struct ReadFileError {
@@ -1327,3 +1321,26 @@ pub(crate) struct MixedExportNameAndNoMangle {
     #[suggestion(style = "verbose", code = "", applicability = "machine-applicable")]
     pub removal_span: Span,
 }
+
+#[derive(Diagnostic, Debug)]
+pub(crate) enum XcrunError {
+    #[diag(codegen_ssa_xcrun_failed_invoking)]
+    FailedInvoking { sdk_name: &'static str, command_formatted: String, error: std::io::Error },
+
+    #[diag(codegen_ssa_xcrun_unsuccessful)]
+    #[note]
+    Unsuccessful {
+        sdk_name: &'static str,
+        command_formatted: String,
+        stdout: String,
+        stderr: String,
+    },
+}
+
+#[derive(Diagnostic, Debug)]
+#[diag(codegen_ssa_xcrun_sdk_path_warning)]
+#[note]
+pub(crate) struct XcrunSdkPathWarning {
+    pub sdk_name: &'static str,
+    pub stderr: String,
+}

Diff for: compiler/rustc_codegen_ssa/src/lib.rs

@@ -13,6 +13,7 @@
 #![feature(let_chains)]
 #![feature(negative_impls)]
 #![feature(rustdoc_internals)]
+#![feature(string_from_utf8_lossy_owned)]
 #![feature(trait_alias)]
 #![feature(try_blocks)]
 // tidy-alphabetical-end