Fix undefined symbol errors when rustup defaults to nightly rust (rust-lang#1771)

* Also set RUSTUP_TOOLCHAIN

Author: tedinski

.github/workflows/kani.yml

@@ -154,6 +154,8 @@ jobs:
             docker run -w /tmp/kani/tests/cargo-kani/build-rs-works  $tag  cargo kani
             docker run $tag  cargo-kani setup --use-local-bundle ./${{ matrix.artifact }}
           done
+          # While the above test OS issues, now try testing with nightly as default:
+          docker run -w /tmp/kani/tests/cargo-kani/simple-lib  kani-20-04  bash -c "rustup default nightly && cargo kani"
 
       # We can't run macos in a container, so we can only test locally.
       # Hopefully any dependency issues won't be unique to macos.

src/lib.rs

@@ -19,7 +19,7 @@ mod setup;
 use std::env;
 use std::ffi::OsString;
 use std::os::unix::prelude::CommandExt;
-use std::path::PathBuf;
+use std::path::{Path, PathBuf};
 use std::process::Command;
 
 use anyhow::{bail, Context, Result};
@@ -77,12 +77,16 @@ fn exec(bin: &str) -> Result<()> {
     let pyroot = kani_dir.join("pyroot");
     let bin_kani = kani_dir.join("bin");
     let bin_pyroot = pyroot.join("bin");
-    let bin_toolchain = kani_dir.join("toolchain").join("bin");
 
     // Allow python scripts to find dependencies under our pyroot
     let pythonpath = prepend_search_path(&[pyroot], env::var_os("PYTHONPATH"))?;
     // Add: kani, cbmc, viewer (pyroot), and our rust toolchain directly to our PATH
-    let path = prepend_search_path(&[bin_kani, bin_pyroot, bin_toolchain], env::var_os("PATH"))?;
+    let path = prepend_search_path(&[bin_kani, bin_pyroot], env::var_os("PATH"))?;
+
+    // Ensure our environment variables for linker search paths won't cause failures, before we execute:
+    fixup_dynamic_linking_environment();
+    // Override our `RUSTUP_TOOLCHAIN` with the version Kani links against
+    set_kani_rust_toolchain(&kani_dir)?;
 
     let mut cmd = Command::new(program);
     cmd.args(env::args_os().skip(1)).env("PYTHONPATH", pythonpath).env("PATH", path).arg0(bin);
@@ -103,3 +107,82 @@ fn prepend_search_path(paths: &[PathBuf], original: Option<OsString>) -> Result<
         }
     }
 }
+
+/// `rustup` sets dynamic linker paths when it proxies to the target Rust toolchain. It's not fully
+/// clear why. `rustup run` exists, which may aid in running Rust binaries that dynamically link to
+/// the Rust standard library with `-C prefer-dynamic`. This might be why. All toolchain binaries
+/// have `RUNPATH` set, so it's not needed by e.g. rustc. (Same for Kani)
+///
+/// However, this causes problems for us when the default Rust toolchain is nightly. Then
+/// `LD_LIBRARY_PATH` is set to a nightly `lib` that may contain a different version of
+/// `librustc_driver-*.so` that might have the same name. This takes priority over the `RUNPATH` of
+/// `kani-compiler` and causes the linker to use a slightly different version of rustc than Kani
+/// was built against. This manifests in errors like:
+/// `kani-compiler: symbol lookup error: ... undefined symbol`
+///
+/// Consequently, let's remove from our linking environment anything that looks like a toolchain
+/// path that rustup set. Then we can safely invoke our binaries. Note also that we update
+/// `PATH` in [`exec`] to include our favored Rust toolchain, so we won't re-drive `rustup` when
+/// `kani-driver` later invokes `cargo`.
+fn fixup_dynamic_linking_environment() {
+    #[cfg(not(target_os = "macos"))]
+    const LOADER_PATH: &str = "LD_LIBRARY_PATH";
+    #[cfg(target_os = "macos")]
+    const LOADER_PATH: &str = "DYLD_FALLBACK_LIBRARY_PATH";
+
+    if let Some(paths) = env::var_os(LOADER_PATH) {
+        // unwrap safety: we're just filtering, so it should always succeed
+        let new_val =
+            env::join_paths(env::split_paths(&paths).filter(unlike_toolchain_path)).unwrap();
+        env::set_var(LOADER_PATH, new_val);
+    }
+}
+
+/// Determines if a path looks unlike a toolchain library path. These often looks like:
+/// `/home/user/.rustup/toolchains/nightly-x86_64-unknown-linux-gnu/lib`
+// Ignore this lint (recommending Path instead of PathBuf),
+// we want to take the right argument type for use in `filter` above.
+#[allow(clippy::ptr_arg)]
+fn unlike_toolchain_path(path: &PathBuf) -> bool {
+    let mut components = path.iter().rev();
+
+    // effectively matching `*/toolchains/*/lib`
+    !(components.next() == Some(std::ffi::OsStr::new("lib"))
+        && components.next().is_some()
+        && components.next() == Some(std::ffi::OsStr::new("toolchains")))
+}
+
+/// We should currently see a `RUSTUP_TOOLCHAIN` that was set by whatever default
+/// toolchain the user has. We override our own environment variable (that is passed
+/// down to children) with the toolchain Kani uses instead.
+fn set_kani_rust_toolchain(kani_dir: &Path) -> Result<()> {
+    let toolchain_verison = setup::get_rust_toolchain_version(kani_dir)?;
+    env::set_var("RUSTUP_TOOLCHAIN", toolchain_verison);
+    Ok(())
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    #[test]
+    fn check_unlike_toolchain_path() {
+        fn trial(s: &str) -> bool {
+            unlike_toolchain_path(&PathBuf::from(s))
+        }
+        // filter these out:
+        assert!(!trial("/home/user/.rustup/toolchains/nightly-x86_64-unknown-linux-gnu/lib"));
+        assert!(!trial("/home/user/.rustup/toolchains/nightly-x86_64-unknown-linux-gnu/lib/"));
+        assert!(!trial("/home/user/.rustup/toolchains/nightly/lib"));
+        assert!(!trial("/home/user/.rustup/toolchains/stable/lib"));
+        // minimally:
+        assert!(!trial("toolchains/nightly/lib"));
+        // keep these:
+        assert!(trial("/home/user/.rustup/toolchains"));
+        assert!(trial("/usr/lib"));
+        assert!(trial("/home/user/lib/toolchains"));
+        // don't error on these:
+        assert!(trial(""));
+        assert!(trial("/"));
+    }
+}

src/setup.rs

@@ -90,11 +90,17 @@ fn setup_kani_bundle(kani_dir: &Path, use_local_bundle: Option<OsString>) -> Res
     Ok(())
 }
 
+/// Reads the Rust toolchain version that Kani was built against from the file in
+/// the Kani release bundle (unpacked in `kani_dir`).
+pub(crate) fn get_rust_toolchain_version(kani_dir: &Path) -> Result<String> {
+    std::fs::read_to_string(kani_dir.join("rust-toolchain-version"))
+        .context("Reading release bundle rust-toolchain-version")
+}
+
 /// Install the Rust toolchain version we require
 fn setup_rust_toolchain(kani_dir: &Path) -> Result<String> {
     // Currently this means we require the bundle to have been unpacked first!
-    let toolchain_version = std::fs::read_to_string(kani_dir.join("rust-toolchain-version"))
-        .context("Reading release bundle rust-toolchain-version")?;
+    let toolchain_version = get_rust_toolchain_version(kani_dir)?;
     println!("[3/5] Installing rust toolchain version: {}", &toolchain_version);
     Command::new("rustup").args(&["toolchain", "install", &toolchain_version]).run()?;