Merge pull request #1357 from bjorn3/build_system_rework6

Support testing of cg_clif in rust's CI

Author: bjorn3

.cirrus.yml

@@ -10,7 +10,7 @@ task:
     folder: target
   prepare_script:
     - . $HOME/.cargo/env
-    - ./y.rs prepare
+    - ./y.sh prepare
   test_script:
     - . $HOME/.cargo/env
-    - ./y.rs test
+    - ./y.sh test

.github/workflows/abi-cafe.yml

@@ -46,12 +46,12 @@ jobs:
       run: rustup set default-host x86_64-pc-windows-gnu
 
     - name: Prepare dependencies
-      run: ./y.rs prepare
+      run: ./y.sh prepare
 
     - name: Build
-      run: ./y.rs build --sysroot none
+      run: ./y.sh build --sysroot none
 
     - name: Test abi-cafe
       env:
         TARGET_TRIPLE: ${{ matrix.env.TARGET_TRIPLE }}
-      run: ./y.rs abi-cafe
+      run: ./y.sh abi-cafe

.github/workflows/main.yml

@@ -19,7 +19,7 @@ jobs:
     - name: Rustfmt
       run: |
         cargo fmt --check
-        rustfmt --check build_system/mod.rs
+        rustfmt --check build_system/main.rs
         rustfmt --check example/*
 
 
@@ -91,15 +91,15 @@ jobs:
         sudo apt-get install -y gcc-s390x-linux-gnu qemu-user
 
     - name: Prepare dependencies
-      run: ./y.rs prepare
+      run: ./y.sh prepare
 
     - name: Build
-      run: ./y.rs build --sysroot none
+      run: ./y.sh build --sysroot none
 
     - name: Test
       env:
         TARGET_TRIPLE: ${{ matrix.env.TARGET_TRIPLE }}
-      run: ./y.rs test
+      run: ./y.sh test
 
     - name: Install LLVM standard library
       run: rustup target add ${{ matrix.env.TARGET_TRIPLE }}
@@ -111,7 +111,7 @@ jobs:
       if: matrix.os != 'windows-latest' || matrix.env.TARGET_TRIPLE != 'x86_64-pc-windows-gnu'
       env:
         TARGET_TRIPLE: ${{ matrix.env.TARGET_TRIPLE }}
-      run: ./y.rs test --sysroot llvm --no-unstable-features
+      run: ./y.sh test --sysroot llvm --no-unstable-features
 
 
   # This job doesn't use cg_clif in any way. It checks that all cg_clif tests work with cg_llvm too.
@@ -165,13 +165,13 @@ jobs:
       run: cargo install hyperfine || true
 
     - name: Prepare dependencies
-      run: ./y.rs prepare
+      run: ./y.sh prepare
 
     - name: Build
-      run: CI_OPT=1 ./y.rs build --sysroot none
+      run: CI_OPT=1 ./y.sh build --sysroot none
 
     - name: Benchmark
-      run: CI_OPT=1 ./y.rs bench
+      run: CI_OPT=1 ./y.sh bench
 
 
   dist:
@@ -224,13 +224,13 @@ jobs:
         sudo apt-get install -y gcc-mingw-w64-x86-64 wine-stable
 
     - name: Prepare dependencies
-      run: ./y.rs prepare
+      run: ./y.sh prepare
 
     - name: Build backend
-      run: CI_OPT=1 ./y.rs build --sysroot none
+      run: CI_OPT=1 ./y.sh build --sysroot none
 
     - name: Build sysroot
-      run: CI_OPT=1 ./y.rs build
+      run: CI_OPT=1 ./y.sh build
 
     - name: Package prebuilt cg_clif
       run: tar cvfJ cg_clif.tar.xz dist

.github/workflows/rustc.yml

@@ -18,7 +18,7 @@ jobs:
         key: ${{ runner.os }}-cargo-build-target-${{ hashFiles('rust-toolchain', '**/Cargo.lock') }}
 
     - name: Prepare dependencies
-      run: ./y.rs prepare
+      run: ./y.sh prepare
 
     - name: Test
       run: ./scripts/test_bootstrap.sh
@@ -38,7 +38,7 @@ jobs:
         key: ${{ runner.os }}-cargo-build-target-${{ hashFiles('rust-toolchain', '**/Cargo.lock') }}
 
     - name: Prepare dependencies
-      run: ./y.rs prepare
+      run: ./y.sh prepare
 
     - name: Test
       run: ./scripts/test_rustc_tests.sh

.gitignore

@@ -1,4 +1,5 @@
 /target
+/build_system/target
 **/*.rs.bk
 *.rlib
 *.o

.vscode/settings.json

@@ -6,9 +6,10 @@
     "rust-analyzer.imports.granularity.enforce": true,
     "rust-analyzer.imports.granularity.group": "module",
     "rust-analyzer.imports.prefix": "crate",
-    "rust-analyzer.cargo.features": ["unstable-features", "__check_build_system_using_ra"],
+    "rust-analyzer.cargo.features": ["unstable-features"],
     "rust-analyzer.linkedProjects": [
         "./Cargo.toml",
+        "./build_system/Cargo.toml",
         {
             "crates": [
                 {

Cargo.toml

@@ -3,13 +3,6 @@ name = "rustc_codegen_cranelift"
 version = "0.1.0"
 edition = "2021"
 
-[[bin]]
-# This is used just to teach rust-analyzer how to check the build system. required-features is used
-# to disable it for regular builds.
-name = "y"
-path = "./y.rs"
-required-features = ["__check_build_system_using_ra"]
-
 [lib]
 crate-type = ["dylib"]
 
@@ -45,7 +38,6 @@ smallvec = "1.8.1"
 unstable-features = ["jit", "inline_asm"]
 jit = ["cranelift-jit", "libloading"]
 inline_asm = []
-__check_build_system_using_ra = []
 
 [package.metadata.rust-analyzer]
 rustc_private = true

Readme.md

@@ -10,8 +10,8 @@ If not please open an issue.
 ```bash
 $ git clone https://github.com/bjorn3/rustc_codegen_cranelift
 $ cd rustc_codegen_cranelift
-$ ./y.rs prepare
-$ ./y.rs build
+$ ./y.sh prepare
+$ ./y.sh build
 ```
 
 To run the test suite replace the last command with:
@@ -20,7 +20,7 @@ To run the test suite replace the last command with:
 $ ./test.sh
 ```
 
-For more docs on how to build and test see [build_system/usage.txt](build_system/usage.txt) or the help message of `./y.rs`.
+For more docs on how to build and test see [build_system/usage.txt](build_system/usage.txt) or the help message of `./y.sh`.
 
 ## Precompiled builds
 
@@ -35,7 +35,7 @@ If you want to use `cargo clif build` instead of having to specify the full path
 
 rustc_codegen_cranelift can be used as a near-drop-in replacement for `cargo build` or `cargo run` for existing projects.
 
-Assuming `$cg_clif_dir` is the directory you cloned this repo into and you followed the instructions (`y.rs prepare` and `y.rs build` or `test.sh`).
+Assuming `$cg_clif_dir` is the directory you cloned this repo into and you followed the instructions (`y.sh prepare` and `y.sh build` or `test.sh`).
 
 In the directory with your project (where you can do the usual `cargo build`), run:
 

build_sysroot/Cargo.toml

@@ -0,0 +1,13 @@
+[package]
+name = "y"
+version = "0.1.0"
+edition = "2021"
+
+[[bin]]
+name = "y"
+path = "main.rs"
+
+[features]
+unstable-features = [] # for rust-analyzer
+
+# Do not add any dependencies

build_sysroot/src/lib.rs

@@ -4,10 +4,15 @@ use super::prepare::GitRepo;
 use super::utils::{spawn_and_wait, CargoProject, Compiler};
 use super::{CodegenBackend, SysrootKind};
 
-static ABI_CAFE_REPO: GitRepo =
-    GitRepo::github("Gankra", "abi-cafe", "4c6dc8c9c687e2b3a760ff2176ce236872b37212", "abi-cafe");
+static ABI_CAFE_REPO: GitRepo = GitRepo::github(
+    "Gankra",
+    "abi-cafe",
+    "4c6dc8c9c687e2b3a760ff2176ce236872b37212",
+    "588df6d66abbe105",
+    "abi-cafe",
+);
 
-static ABI_CAFE: CargoProject = CargoProject::new(&ABI_CAFE_REPO.source_dir(), "abi_cafe");
+static ABI_CAFE: CargoProject = CargoProject::new(&ABI_CAFE_REPO.source_dir(), "abi_cafe_target");
 
 pub(crate) fn run(
     channel: &str,
@@ -18,6 +23,7 @@ pub(crate) fn run(
     bootstrap_host_compiler: &Compiler,
 ) {
     ABI_CAFE_REPO.fetch(dirs);
+    ABI_CAFE_REPO.patch(dirs);
 
     eprintln!("Building sysroot for abi-cafe");
     build_sysroot::build_sysroot(

build_system/Cargo.lock

@@ -10,6 +10,7 @@ static SIMPLE_RAYTRACER_REPO: GitRepo = GitRepo::github(
     "ebobby",
     "simple-raytracer",
     "804a7a21b9e673a482797aa289a18ed480e4d813",
+    "ad6f59a2331a3f56",
     "<none>",
 );
 
@@ -24,9 +25,8 @@ fn benchmark_simple_raytracer(dirs: &Dirs, bootstrap_host_compiler: &Compiler) {
         std::process::exit(1);
     }
 
-    if !SIMPLE_RAYTRACER_REPO.source_dir().to_path(dirs).exists() {
-        SIMPLE_RAYTRACER_REPO.fetch(dirs);
-    }
+    SIMPLE_RAYTRACER_REPO.fetch(dirs);
+    SIMPLE_RAYTRACER_REPO.patch(dirs);
 
     let bench_runs = env::var("BENCH_RUNS").unwrap_or_else(|_| "10".to_string()).parse().unwrap();
 

build_system/Cargo.toml

@@ -3,7 +3,7 @@ use std::path::PathBuf;
 
 use super::path::{Dirs, RelPath};
 use super::rustc_info::get_file_name;
-use super::utils::{is_ci, is_ci_opt, CargoProject, Compiler};
+use super::utils::{is_ci, is_ci_opt, maybe_incremental, CargoProject, Compiler};
 
 pub(crate) static CG_CLIF: CargoProject = CargoProject::new(&RelPath::SOURCE, "cg_clif");
 
@@ -14,18 +14,14 @@ pub(crate) fn build_backend(
     use_unstable_features: bool,
 ) -> PathBuf {
     let mut cmd = CG_CLIF.build(&bootstrap_host_compiler, dirs);
-
-    cmd.env("CARGO_BUILD_INCREMENTAL", "true"); // Force incr comp even in release mode
+    maybe_incremental(&mut cmd);
 
     let mut rustflags = env::var("RUSTFLAGS").unwrap_or_default();
 
     if is_ci() {
         // Deny warnings on CI
         rustflags += " -Dwarnings";
 
-        // Disabling incr comp reduces cache size and incr comp doesn't save as much on CI anyway
-        cmd.env("CARGO_BUILD_INCREMENTAL", "false");
-
         if !is_ci_opt() {
             cmd.env("CARGO_PROFILE_RELEASE_DEBUG_ASSERTIONS", "true");
             cmd.env("CARGO_PROFILE_RELEASE_OVERFLOW_CHECKS", "true");