Auto merge of rust-lang#107843 - bjorn3:sync_cg_clif-2023-02-09, r=bjorn3

Sync rustc_codegen_cranelift

* Couple of bugfixes
* A significant runtime perf improvement
* Implemented sym and const support for inline asm
* Improved self profile integration

r? `@ghost`

`@rustbot` label +A-codegen +A-cranelift +T-compiler

Author: bors

compiler/rustc_codegen_cranelift/.github/workflows/main.yml

@@ -21,6 +21,7 @@ jobs:
         cargo fmt --check
         rustfmt --check build_system/mod.rs
 
+
   build:
     runs-on: ${{ matrix.os }}
     timeout-minutes: 60
@@ -33,7 +34,7 @@ jobs:
       fail-fast: false
       matrix:
         include:
-          - os: ubuntu-20.04 # FIXME switch to ubuntu-22.04 once #1303 is fixed
+          - os: ubuntu-latest
             env:
               TARGET_TRIPLE: x86_64-unknown-linux-gnu
           - os: macos-latest
@@ -112,23 +113,6 @@ jobs:
         TARGET_TRIPLE: ${{ matrix.env.TARGET_TRIPLE }}
       run: ./y.rs test
 
-    - name: Package prebuilt cg_clif
-      run: tar cvfJ cg_clif.tar.xz dist
-
-    - name: Upload prebuilt cg_clif
-      if: matrix.os == 'windows-latest' || matrix.env.TARGET_TRIPLE != 'x86_64-pc-windows-gnu'
-      uses: actions/upload-artifact@v3
-      with:
-        name: cg_clif-${{ matrix.env.TARGET_TRIPLE }}
-        path: cg_clif.tar.xz
-
-    - name: Upload prebuilt cg_clif (cross compile)
-      if: matrix.os != 'windows-latest' && matrix.env.TARGET_TRIPLE == 'x86_64-pc-windows-gnu'
-      uses: actions/upload-artifact@v3
-      with:
-        name: cg_clif-${{ runner.os }}-cross-x86_64-mingw
-        path: cg_clif.tar.xz
-
 
   abi_cafe:
     runs-on: ${{ matrix.os }}
@@ -185,3 +169,129 @@ jobs:
       env:
         TARGET_TRIPLE: ${{ matrix.env.TARGET_TRIPLE }}
       run: ./y.rs abi-cafe
+
+
+  bench:
+    runs-on: ubuntu-latest
+    timeout-minutes: 60
+
+    defaults:
+      run:
+        shell: bash
+
+    steps:
+    - uses: actions/checkout@v3
+
+    - name: Cache cargo target dir
+      uses: actions/cache@v3
+      with:
+        path: build/cg_clif
+        key: ${{ runner.os }}-x86_64-unknown-linux-gnu-cargo-build-target-${{ hashFiles('rust-toolchain', '**/Cargo.lock') }}
+
+    - name: Cache cargo bin dir
+      uses: actions/cache@v3
+      with:
+        path: ~/.cargo/bin
+        key: ${{ runner.os }}-${{ matrix.env.TARGET_TRIPLE }}-cargo-bin-dir-${{ hashFiles('rust-toolchain', '**/Cargo.lock') }}
+
+    - name: Use sparse cargo registry
+      run: |
+        cat >> ~/.cargo/config.toml <<EOF
+        [unstable]
+        sparse-registry = true
+        EOF
+
+    - name: Install hyperfine
+      run: cargo install hyperfine || true
+
+    - name: Prepare dependencies
+      run: ./y.rs prepare
+
+    - name: Build
+      run: CI_OPT=1 ./y.rs build --sysroot none
+
+    - name: Benchmark
+      run: CI_OPT=1 ./y.rs bench
+
+
+  dist:
+    runs-on: ${{ matrix.os }}
+    timeout-minutes: 60
+
+    defaults:
+      run:
+        shell: bash
+
+    strategy:
+      fail-fast: false
+      matrix:
+        include:
+          # FIXME update at some point in the future once most distros use a newer glibc
+          - os: ubuntu-20.04
+            env:
+              TARGET_TRIPLE: x86_64-unknown-linux-gnu
+          - os: macos-latest
+            env:
+              TARGET_TRIPLE: x86_64-apple-darwin
+          # cross-compile from Linux to Windows using mingw
+          - os: ubuntu-latest
+            env:
+              TARGET_TRIPLE: x86_64-pc-windows-gnu
+          - os: windows-latest
+            env:
+              TARGET_TRIPLE: x86_64-pc-windows-msvc
+          - os: windows-latest
+            env:
+              TARGET_TRIPLE: x86_64-pc-windows-gnu
+
+    steps:
+    - uses: actions/checkout@v3
+
+    - name: Cache cargo target dir
+      uses: actions/cache@v3
+      with:
+        path: build/cg_clif
+        key: ${{ runner.os }}-${{ matrix.env.TARGET_TRIPLE }}-dist-cargo-build-target-${{ hashFiles('rust-toolchain', '**/Cargo.lock') }}
+
+    - name: Set MinGW as the default toolchain
+      if: matrix.os == 'windows-latest' && matrix.env.TARGET_TRIPLE == 'x86_64-pc-windows-gnu'
+      run: rustup set default-host x86_64-pc-windows-gnu
+
+    - name: Install MinGW toolchain and wine
+      if: matrix.os == 'ubuntu-latest' && matrix.env.TARGET_TRIPLE == 'x86_64-pc-windows-gnu'
+      run: |
+        sudo apt-get update
+        sudo apt-get install -y gcc-mingw-w64-x86-64 wine-stable
+
+    - name: Use sparse cargo registry
+      run: |
+        cat >> ~/.cargo/config.toml <<EOF
+        [unstable]
+        sparse-registry = true
+        EOF
+
+    - name: Prepare dependencies
+      run: ./y.rs prepare
+
+    - name: Build backend
+      run: CI_OPT=1 ./y.rs build --sysroot none
+
+    - name: Build sysroot
+      run: CI_OPT=1 ./y.rs build
+
+    - name: Package prebuilt cg_clif
+      run: tar cvfJ cg_clif.tar.xz dist
+
+    - name: Upload prebuilt cg_clif
+      if: matrix.os == 'windows-latest' || matrix.env.TARGET_TRIPLE != 'x86_64-pc-windows-gnu'
+      uses: actions/upload-artifact@v3
+      with:
+        name: cg_clif-${{ matrix.env.TARGET_TRIPLE }}
+        path: cg_clif.tar.xz
+
+    - name: Upload prebuilt cg_clif (cross compile)
+      if: matrix.os != 'windows-latest' && matrix.env.TARGET_TRIPLE == 'x86_64-pc-windows-gnu'
+      uses: actions/upload-artifact@v3
+      with:
+        name: cg_clif-${{ runner.os }}-cross-x86_64-mingw
+        path: cg_clif.tar.xz

compiler/rustc_codegen_cranelift/build_sysroot/Cargo.lock

@@ -34,9 +34,9 @@ dependencies = [
 
 [[package]]
 name = "cc"
-version = "1.0.78"
+version = "1.0.79"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "a20104e2335ce8a659d6dd92a51a767a0c062599c73b343fd152cb401e828c3d"
+checksum = "50d30906286121d95be3d479533b458f87493b30a4b5f79a607db8f5d11aa91f"
 
 [[package]]
 name = "cfg-if"

compiler/rustc_codegen_cranelift/build_system/abi_cafe.rs

@@ -6,11 +6,10 @@ use super::prepare::GitRepo;
 use super::utils::{spawn_and_wait, CargoProject, Compiler};
 use super::SysrootKind;
 
-pub(crate) static ABI_CAFE_REPO: GitRepo =
+static ABI_CAFE_REPO: GitRepo =
     GitRepo::github("Gankra", "abi-cafe", "4c6dc8c9c687e2b3a760ff2176ce236872b37212", "abi-cafe");
 
-pub(crate) static ABI_CAFE: CargoProject =
-    CargoProject::new(&ABI_CAFE_REPO.source_dir(), "abi_cafe");
+static ABI_CAFE: CargoProject = CargoProject::new(&ABI_CAFE_REPO.source_dir(), "abi_cafe");
 
 pub(crate) fn run(
     channel: &str,
@@ -19,6 +18,9 @@ pub(crate) fn run(
     cg_clif_dylib: &Path,
     bootstrap_host_compiler: &Compiler,
 ) {
+    ABI_CAFE_REPO.fetch(dirs);
+    spawn_and_wait(ABI_CAFE.fetch("cargo", &bootstrap_host_compiler.rustc, dirs));
+
     eprintln!("Building sysroot for abi-cafe");
     build_sysroot::build_sysroot(
         dirs,

compiler/rustc_codegen_cranelift/build_system/bench.rs

@@ -5,20 +5,20 @@ use std::path::Path;
 use super::path::{Dirs, RelPath};
 use super::prepare::GitRepo;
 use super::rustc_info::get_file_name;
-use super::utils::{hyperfine_command, is_ci, spawn_and_wait, CargoProject, Compiler};
+use super::utils::{hyperfine_command, spawn_and_wait, CargoProject, Compiler};
 
-pub(crate) static SIMPLE_RAYTRACER_REPO: GitRepo = GitRepo::github(
+static SIMPLE_RAYTRACER_REPO: GitRepo = GitRepo::github(
     "ebobby",
     "simple-raytracer",
     "804a7a21b9e673a482797aa289a18ed480e4d813",
     "<none>",
 );
 
 // Use a separate target dir for the initial LLVM build to reduce unnecessary recompiles
-pub(crate) static SIMPLE_RAYTRACER_LLVM: CargoProject =
+static SIMPLE_RAYTRACER_LLVM: CargoProject =
     CargoProject::new(&SIMPLE_RAYTRACER_REPO.source_dir(), "simple_raytracer_llvm");
 
-pub(crate) static SIMPLE_RAYTRACER: CargoProject =
+static SIMPLE_RAYTRACER: CargoProject =
     CargoProject::new(&SIMPLE_RAYTRACER_REPO.source_dir(), "simple_raytracer");
 
 pub(crate) fn benchmark(dirs: &Dirs, bootstrap_host_compiler: &Compiler) {
@@ -32,6 +32,15 @@ fn benchmark_simple_raytracer(dirs: &Dirs, bootstrap_host_compiler: &Compiler) {
         std::process::exit(1);
     }
 
+    if !SIMPLE_RAYTRACER_REPO.source_dir().to_path(dirs).exists() {
+        SIMPLE_RAYTRACER_REPO.fetch(dirs);
+        spawn_and_wait(SIMPLE_RAYTRACER.fetch(
+            &bootstrap_host_compiler.cargo,
+            &bootstrap_host_compiler.rustc,
+            dirs,
+        ));
+    }
+
     eprintln!("[LLVM BUILD] simple-raytracer");
     let build_cmd = SIMPLE_RAYTRACER_LLVM.build(bootstrap_host_compiler, dirs);
     spawn_and_wait(build_cmd);
@@ -45,10 +54,7 @@ fn benchmark_simple_raytracer(dirs: &Dirs, bootstrap_host_compiler: &Compiler) {
     )
     .unwrap();
 
-    let run_runs = env::var("RUN_RUNS")
-        .unwrap_or(if is_ci() { "2" } else { "10" }.to_string())
-        .parse()
-        .unwrap();
+    let bench_runs = env::var("BENCH_RUNS").unwrap_or_else(|_| "10".to_string()).parse().unwrap();
 
     eprintln!("[BENCH COMPILE] ebobby/simple-raytracer");
     let cargo_clif =
@@ -57,24 +63,24 @@ fn benchmark_simple_raytracer(dirs: &Dirs, bootstrap_host_compiler: &Compiler) {
     let target_dir = SIMPLE_RAYTRACER.target_dir(dirs);
 
     let clean_cmd = format!(
-        "cargo clean --manifest-path {manifest_path} --target-dir {target_dir}",
+        "RUSTC=rustc cargo clean --manifest-path {manifest_path} --target-dir {target_dir}",
         manifest_path = manifest_path.display(),
         target_dir = target_dir.display(),
     );
     let llvm_build_cmd = format!(
-        "cargo build --manifest-path {manifest_path} --target-dir {target_dir}",
+        "RUSTC=rustc cargo build --manifest-path {manifest_path} --target-dir {target_dir}",
         manifest_path = manifest_path.display(),
         target_dir = target_dir.display(),
     );
     let clif_build_cmd = format!(
-        "{cargo_clif} build --manifest-path {manifest_path} --target-dir {target_dir}",
+        "RUSTC=rustc {cargo_clif} build --manifest-path {manifest_path} --target-dir {target_dir}",
         cargo_clif = cargo_clif.display(),
         manifest_path = manifest_path.display(),
         target_dir = target_dir.display(),
     );
 
     let bench_compile =
-        hyperfine_command(1, run_runs, Some(&clean_cmd), &llvm_build_cmd, &clif_build_cmd);
+        hyperfine_command(1, bench_runs, Some(&clean_cmd), &llvm_build_cmd, &clif_build_cmd);
 
     spawn_and_wait(bench_compile);
 
@@ -87,7 +93,7 @@ fn benchmark_simple_raytracer(dirs: &Dirs, bootstrap_host_compiler: &Compiler) {
 
     let mut bench_run = hyperfine_command(
         0,
-        run_runs,
+        bench_runs,
         None,
         Path::new(".").join(get_file_name("raytracer_cg_llvm", "bin")).to_str().unwrap(),
         Path::new(".").join(get_file_name("raytracer_cg_clif", "bin")).to_str().unwrap(),

compiler/rustc_codegen_cranelift/build_system/build_backend.rs

@@ -3,7 +3,7 @@ use std::path::PathBuf;
 
 use super::path::{Dirs, RelPath};
 use super::rustc_info::get_file_name;
-use super::utils::{is_ci, CargoProject, Compiler};
+use super::utils::{is_ci, is_ci_opt, CargoProject, Compiler};
 
 pub(crate) static CG_CLIF: CargoProject = CargoProject::new(&RelPath::SOURCE, "cg_clif");
 
@@ -26,7 +26,9 @@ pub(crate) fn build_backend(
         // Disabling incr comp reduces cache size and incr comp doesn't save as much on CI anyway
         cmd.env("CARGO_BUILD_INCREMENTAL", "false");
 
-        cmd.env("CARGO_PROFILE_RELEASE_DEBUG_ASSERTIONS", "true");
+        if !is_ci_opt() {
+            cmd.env("CARGO_PROFILE_RELEASE_DEBUG_ASSERTIONS", "true");
+        }
     }
 
     if use_unstable_features {

compiler/rustc_codegen_cranelift/build_system/build_sysroot.rs

@@ -248,6 +248,9 @@ fn build_clif_sysroot_for_triple(
         build_cmd.arg("--release");
     }
     build_cmd.env("__CARGO_DEFAULT_LIB_METADATA", "cg_clif");
+    if compiler.triple.contains("apple") {
+        build_cmd.env("CARGO_PROFILE_RELEASE_SPLIT_DEBUGINFO", "packed");
+    }
     spawn_and_wait(build_cmd);
 
     for entry in fs::read_dir(build_dir.join("deps")).unwrap() {

compiler/rustc_codegen_cranelift/build_system/mod.rs

@@ -2,7 +2,7 @@ use std::env;
 use std::path::PathBuf;
 use std::process;
 
-use self::utils::{is_ci, Compiler};
+use self::utils::{is_ci, is_ci_opt, Compiler};
 
 mod abi_cafe;
 mod bench;
@@ -53,8 +53,10 @@ pub fn main() {
         // Disabling incr comp reduces cache size and incr comp doesn't save as much on CI anyway
         env::set_var("CARGO_BUILD_INCREMENTAL", "false");
 
-        // Enable the Cranelift verifier
-        env::set_var("CG_CLIF_ENABLE_VERIFIER", "1");
+        if !is_ci_opt() {
+            // Enable the Cranelift verifier
+            env::set_var("CG_CLIF_ENABLE_VERIFIER", "1");
+        }
     }
 
     let mut args = env::args().skip(1);

compiler/rustc_codegen_cranelift/build_system/prepare.rs

@@ -11,22 +11,18 @@ use super::utils::{copy_dir_recursively, git_command, retry_spawn_and_wait, spaw
 pub(crate) fn prepare(dirs: &Dirs) {
     RelPath::DOWNLOAD.ensure_fresh(dirs);
 
-    spawn_and_wait(super::build_backend::CG_CLIF.fetch("cargo", dirs));
+    spawn_and_wait(super::build_backend::CG_CLIF.fetch("cargo", "rustc", dirs));
 
     prepare_sysroot(dirs);
-    spawn_and_wait(super::build_sysroot::STANDARD_LIBRARY.fetch("cargo", dirs));
-    spawn_and_wait(super::tests::LIBCORE_TESTS.fetch("cargo", dirs));
+    spawn_and_wait(super::build_sysroot::STANDARD_LIBRARY.fetch("cargo", "rustc", dirs));
+    spawn_and_wait(super::tests::LIBCORE_TESTS.fetch("cargo", "rustc", dirs));
 
-    super::abi_cafe::ABI_CAFE_REPO.fetch(dirs);
-    spawn_and_wait(super::abi_cafe::ABI_CAFE.fetch("cargo", dirs));
     super::tests::RAND_REPO.fetch(dirs);
-    spawn_and_wait(super::tests::RAND.fetch("cargo", dirs));
+    spawn_and_wait(super::tests::RAND.fetch("cargo", "rustc", dirs));
     super::tests::REGEX_REPO.fetch(dirs);
-    spawn_and_wait(super::tests::REGEX.fetch("cargo", dirs));
+    spawn_and_wait(super::tests::REGEX.fetch("cargo", "rustc", dirs));
     super::tests::PORTABLE_SIMD_REPO.fetch(dirs);
-    spawn_and_wait(super::tests::PORTABLE_SIMD.fetch("cargo", dirs));
-    super::bench::SIMPLE_RAYTRACER_REPO.fetch(dirs);
-    spawn_and_wait(super::bench::SIMPLE_RAYTRACER.fetch("cargo", dirs));
+    spawn_and_wait(super::tests::PORTABLE_SIMD.fetch("cargo", "rustc", dirs));
 }
 
 fn prepare_sysroot(dirs: &Dirs) {
@@ -80,7 +76,7 @@ impl GitRepo {
         }
     }
 
-    fn fetch(&self, dirs: &Dirs) {
+    pub(crate) fn fetch(&self, dirs: &Dirs) {
         match self.url {
             GitRepoUrl::Github { user, repo } => {
                 clone_repo_shallow_github(