From ce3f300e40146480d3f508e9657c454bf6881b10 Mon Sep 17 00:00:00 2001 From: bjorn3 <17426603+bjorn3@users.noreply.github.com> Date: Mon, 13 Feb 2023 10:38:25 +0000 Subject: [PATCH 01/20] Add --download-dir option to specify download dir separate from --out-dir --- build_system/mod.rs | 12 ++++++++++-- build_system/usage.txt | 13 ++++++++----- 2 files changed, 18 insertions(+), 7 deletions(-) diff --git a/build_system/mod.rs b/build_system/mod.rs index 011001116..e1b46c312 100644 --- a/build_system/mod.rs +++ b/build_system/mod.rs @@ -81,6 +81,7 @@ pub(crate) fn main() { }; let mut out_dir = PathBuf::from("."); + let mut download_dir = None; let mut channel = "release"; let mut sysroot_kind = SysrootKind::Clif; let mut use_unstable_features = true; @@ -91,7 +92,12 @@ pub(crate) fn main() { "--out-dir" => { out_dir = PathBuf::from(args.next().unwrap_or_else(|| { arg_error!("--out-dir requires argument"); - })) + })); + } + "--download-dir" => { + download_dir = Some(PathBuf::from(args.next().unwrap_or_else(|| { + arg_error!("--download-dir requires argument"); + }))); } "--debug" => channel = "debug", "--sysroot" => { @@ -152,7 +158,9 @@ pub(crate) fn main() { out_dir = current_dir.join(out_dir); let dirs = path::Dirs { source_dir: current_dir.clone(), - download_dir: out_dir.join("download"), + download_dir: download_dir + .map(|dir| current_dir.join(dir)) + .unwrap_or_else(|| out_dir.join("download")), build_dir: out_dir.join("build"), dist_dir: out_dir.join("dist"), frozen, diff --git a/build_system/usage.txt b/build_system/usage.txt index 3bc18a933..98726ae5a 100644 --- a/build_system/usage.txt +++ b/build_system/usage.txt @@ -1,11 +1,11 @@ The build system of cg_clif. USAGE: - ./y.rs prepare [--out-dir DIR] - ./y.rs build [--debug] [--sysroot none|clif|llvm] [--out-dir DIR] [--no-unstable-features] [--frozen] - ./y.rs test [--debug] [--sysroot none|clif|llvm] [--out-dir DIR] [--no-unstable-features] [--frozen] - ./y.rs abi-cafe [--debug] [--sysroot none|clif|llvm] [--out-dir DIR] [--no-unstable-features] [--frozen] - ./y.rs bench [--debug] [--sysroot none|clif|llvm] [--out-dir DIR] [--no-unstable-features] [--frozen] + ./y.rs prepare [--out-dir DIR] [--download-dir DIR] + ./y.rs build [--debug] [--sysroot none|clif|llvm] [--out-dir DIR] [--download-dir DIR] [--no-unstable-features] [--frozen] + ./y.rs test [--debug] [--sysroot none|clif|llvm] [--out-dir DIR] [--download-dir DIR] [--no-unstable-features] [--frozen] + ./y.rs abi-cafe [--debug] [--sysroot none|clif|llvm] [--out-dir DIR] [--download-dir DIR] [--no-unstable-features] [--frozen] + ./y.rs bench [--debug] [--sysroot none|clif|llvm] [--out-dir DIR] [--download-dir DIR] [--no-unstable-features] [--frozen] OPTIONS: --debug @@ -22,6 +22,9 @@ OPTIONS: Specify the directory in which the download, build and dist directories are stored. By default this is the working directory. + --download-dir DIR + Specify the directory in which the download directory is stored. Overrides --out-dir. + --no-unstable-features Some features are not yet ready for production usage. This option will disable these features. This includes the JIT mode and inline assembly support. From d0b8896189f5b2fb472d54ed389681dff97d907b Mon Sep 17 00:00:00 2001 From: bjorn3 <17426603+bjorn3@users.noreply.github.com> Date: Wed, 15 Feb 2023 18:23:11 +0000 Subject: [PATCH 02/20] Allow building the build system using cargo Rust's build system only handles cargo, not rustc. --- .cirrus.yml | 4 ++-- .github/workflows/abi-cafe.yml | 6 +++--- .github/workflows/main.yml | 22 ++++++++++----------- .github/workflows/rustc.yml | 4 ++-- .gitignore | 1 + .vscode/settings.json | 3 ++- Cargo.toml | 8 -------- Readme.md | 8 ++++---- build_system/Cargo.lock | 7 +++++++ build_system/Cargo.toml | 13 +++++++++++++ build_system/build_sysroot.rs | 4 ++-- build_system/{mod.rs => main.rs} | 10 +++++++--- build_system/usage.txt | 16 ++++++++-------- clean_all.sh | 2 +- docs/usage.md | 2 +- scripts/rustup.sh | 2 +- scripts/setup_rust_fork.sh | 2 +- test.sh | 2 +- y.rs | 33 ++------------------------------ y.sh | 6 ++++++ 20 files changed, 75 insertions(+), 80 deletions(-) create mode 100644 build_system/Cargo.lock create mode 100644 build_system/Cargo.toml rename build_system/{mod.rs => main.rs} (98%) create mode 100755 y.sh diff --git a/.cirrus.yml b/.cirrus.yml index 1405ea749..8b4efd4e3 100644 --- a/.cirrus.yml +++ b/.cirrus.yml @@ -10,7 +10,7 @@ task: folder: target prepare_script: - . $HOME/.cargo/env - - ./y.rs prepare + - ./y.sh prepare test_script: - . $HOME/.cargo/env - - ./y.rs test + - ./y.sh test diff --git a/.github/workflows/abi-cafe.yml b/.github/workflows/abi-cafe.yml index 3c4055566..12aa69d3c 100644 --- a/.github/workflows/abi-cafe.yml +++ b/.github/workflows/abi-cafe.yml @@ -46,12 +46,12 @@ jobs: run: rustup set default-host x86_64-pc-windows-gnu - name: Prepare dependencies - run: ./y.rs prepare + run: ./y.sh prepare - name: Build - run: ./y.rs build --sysroot none + run: ./y.sh build --sysroot none - name: Test abi-cafe env: TARGET_TRIPLE: ${{ matrix.env.TARGET_TRIPLE }} - run: ./y.rs abi-cafe + run: ./y.sh abi-cafe diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index abcd1affd..8e6c1e8ad 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -19,7 +19,7 @@ jobs: - name: Rustfmt run: | cargo fmt --check - rustfmt --check build_system/mod.rs + rustfmt --check build_system/main.rs rustfmt --check example/* @@ -91,15 +91,15 @@ jobs: sudo apt-get install -y gcc-s390x-linux-gnu qemu-user - name: Prepare dependencies - run: ./y.rs prepare + run: ./y.sh prepare - name: Build - run: ./y.rs build --sysroot none + run: ./y.sh build --sysroot none - name: Test env: TARGET_TRIPLE: ${{ matrix.env.TARGET_TRIPLE }} - run: ./y.rs test + run: ./y.sh test - name: Install LLVM standard library run: rustup target add ${{ matrix.env.TARGET_TRIPLE }} @@ -111,7 +111,7 @@ jobs: if: matrix.os != 'windows-latest' || matrix.env.TARGET_TRIPLE != 'x86_64-pc-windows-gnu' env: TARGET_TRIPLE: ${{ matrix.env.TARGET_TRIPLE }} - run: ./y.rs test --sysroot llvm --no-unstable-features + run: ./y.sh test --sysroot llvm --no-unstable-features # This job doesn't use cg_clif in any way. It checks that all cg_clif tests work with cg_llvm too. @@ -165,13 +165,13 @@ jobs: run: cargo install hyperfine || true - name: Prepare dependencies - run: ./y.rs prepare + run: ./y.sh prepare - name: Build - run: CI_OPT=1 ./y.rs build --sysroot none + run: CI_OPT=1 ./y.sh build --sysroot none - name: Benchmark - run: CI_OPT=1 ./y.rs bench + run: CI_OPT=1 ./y.sh bench dist: @@ -224,13 +224,13 @@ jobs: sudo apt-get install -y gcc-mingw-w64-x86-64 wine-stable - name: Prepare dependencies - run: ./y.rs prepare + run: ./y.sh prepare - name: Build backend - run: CI_OPT=1 ./y.rs build --sysroot none + run: CI_OPT=1 ./y.sh build --sysroot none - name: Build sysroot - run: CI_OPT=1 ./y.rs build + run: CI_OPT=1 ./y.sh build - name: Package prebuilt cg_clif run: tar cvfJ cg_clif.tar.xz dist diff --git a/.github/workflows/rustc.yml b/.github/workflows/rustc.yml index b2f772c4f..b49dc3aff 100644 --- a/.github/workflows/rustc.yml +++ b/.github/workflows/rustc.yml @@ -18,7 +18,7 @@ jobs: key: ${{ runner.os }}-cargo-build-target-${{ hashFiles('rust-toolchain', '**/Cargo.lock') }} - name: Prepare dependencies - run: ./y.rs prepare + run: ./y.sh prepare - name: Test run: ./scripts/test_bootstrap.sh @@ -38,7 +38,7 @@ jobs: key: ${{ runner.os }}-cargo-build-target-${{ hashFiles('rust-toolchain', '**/Cargo.lock') }} - name: Prepare dependencies - run: ./y.rs prepare + run: ./y.sh prepare - name: Test run: ./scripts/test_rustc_tests.sh diff --git a/.gitignore b/.gitignore index e5d10a937..e6ac8c840 100644 --- a/.gitignore +++ b/.gitignore @@ -1,4 +1,5 @@ /target +/build_system/target **/*.rs.bk *.rlib *.o diff --git a/.vscode/settings.json b/.vscode/settings.json index 7c8703cba..60cb51d56 100644 --- a/.vscode/settings.json +++ b/.vscode/settings.json @@ -6,9 +6,10 @@ "rust-analyzer.imports.granularity.enforce": true, "rust-analyzer.imports.granularity.group": "module", "rust-analyzer.imports.prefix": "crate", - "rust-analyzer.cargo.features": ["unstable-features", "__check_build_system_using_ra"], + "rust-analyzer.cargo.features": ["unstable-features"], "rust-analyzer.linkedProjects": [ "./Cargo.toml", + "./build_system/Cargo.toml", { "crates": [ { diff --git a/Cargo.toml b/Cargo.toml index 145b11504..1c1f2d857 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -3,13 +3,6 @@ name = "rustc_codegen_cranelift" version = "0.1.0" edition = "2021" -[[bin]] -# This is used just to teach rust-analyzer how to check the build system. required-features is used -# to disable it for regular builds. -name = "y" -path = "./y.rs" -required-features = ["__check_build_system_using_ra"] - [lib] crate-type = ["dylib"] @@ -45,7 +38,6 @@ smallvec = "1.8.1" unstable-features = ["jit", "inline_asm"] jit = ["cranelift-jit", "libloading"] inline_asm = [] -__check_build_system_using_ra = [] [package.metadata.rust-analyzer] rustc_private = true diff --git a/Readme.md b/Readme.md index 7e9945578..9469feea0 100644 --- a/Readme.md +++ b/Readme.md @@ -10,8 +10,8 @@ If not please open an issue. ```bash $ git clone https://github.com/bjorn3/rustc_codegen_cranelift $ cd rustc_codegen_cranelift -$ ./y.rs prepare -$ ./y.rs build +$ ./y.sh prepare +$ ./y.sh build ``` To run the test suite replace the last command with: @@ -20,7 +20,7 @@ To run the test suite replace the last command with: $ ./test.sh ``` -For more docs on how to build and test see [build_system/usage.txt](build_system/usage.txt) or the help message of `./y.rs`. +For more docs on how to build and test see [build_system/usage.txt](build_system/usage.txt) or the help message of `./y.sh`. ## Precompiled builds @@ -35,7 +35,7 @@ If you want to use `cargo clif build` instead of having to specify the full path rustc_codegen_cranelift can be used as a near-drop-in replacement for `cargo build` or `cargo run` for existing projects. -Assuming `$cg_clif_dir` is the directory you cloned this repo into and you followed the instructions (`y.rs prepare` and `y.rs build` or `test.sh`). +Assuming `$cg_clif_dir` is the directory you cloned this repo into and you followed the instructions (`y.sh prepare` and `y.sh build` or `test.sh`). In the directory with your project (where you can do the usual `cargo build`), run: diff --git a/build_system/Cargo.lock b/build_system/Cargo.lock new file mode 100644 index 000000000..86268e191 --- /dev/null +++ b/build_system/Cargo.lock @@ -0,0 +1,7 @@ +# This file is automatically @generated by Cargo. +# It is not intended for manual editing. +version = 3 + +[[package]] +name = "y" +version = "0.1.0" diff --git a/build_system/Cargo.toml b/build_system/Cargo.toml new file mode 100644 index 000000000..f47b9bc55 --- /dev/null +++ b/build_system/Cargo.toml @@ -0,0 +1,13 @@ +[package] +name = "y" +version = "0.1.0" +edition = "2021" + +[[bin]] +name = "y" +path = "main.rs" + +[features] +unstable-features = [] # for rust-analyzer + +# Do not add any dependencies diff --git a/build_system/build_sysroot.rs b/build_system/build_sysroot.rs index dab9c77d1..2ed6272b2 100644 --- a/build_system/build_sysroot.rs +++ b/build_system/build_sysroot.rs @@ -225,7 +225,7 @@ fn build_clif_sysroot_for_triple( match fs::read_to_string(SYSROOT_RUSTC_VERSION.to_path(dirs)) { Err(e) => { eprintln!("Failed to get rustc version for patched sysroot source: {}", e); - eprintln!("Hint: Try `./y.rs prepare` to patch the sysroot source"); + eprintln!("Hint: Try `./y.sh prepare` to patch the sysroot source"); process::exit(1); } Ok(source_version) => { @@ -234,7 +234,7 @@ fn build_clif_sysroot_for_triple( eprintln!("The patched sysroot source is outdated"); eprintln!("Source version: {}", source_version.trim()); eprintln!("Rustc version: {}", rustc_version.trim()); - eprintln!("Hint: Try `./y.rs prepare` to update the patched sysroot source"); + eprintln!("Hint: Try `./y.sh prepare` to update the patched sysroot source"); process::exit(1); } } diff --git a/build_system/mod.rs b/build_system/main.rs similarity index 98% rename from build_system/mod.rs rename to build_system/main.rs index e1b46c312..06395eb14 100644 --- a/build_system/mod.rs +++ b/build_system/main.rs @@ -1,3 +1,7 @@ +#![warn(rust_2018_idioms)] +#![warn(unused_lifetimes)] +#![warn(unreachable_pub)] + use std::env; use std::path::PathBuf; use std::process; @@ -37,19 +41,19 @@ enum Command { } #[derive(Copy, Clone, Debug)] -pub(crate) enum SysrootKind { +enum SysrootKind { None, Clif, Llvm, } #[derive(Clone, Debug)] -pub(crate) enum CodegenBackend { +enum CodegenBackend { Local(PathBuf), Builtin(String), } -pub(crate) fn main() { +fn main() { if env::var("RUST_BACKTRACE").is_err() { env::set_var("RUST_BACKTRACE", "1"); } diff --git a/build_system/usage.txt b/build_system/usage.txt index 98726ae5a..9d20cdca6 100644 --- a/build_system/usage.txt +++ b/build_system/usage.txt @@ -1,11 +1,11 @@ The build system of cg_clif. USAGE: - ./y.rs prepare [--out-dir DIR] [--download-dir DIR] - ./y.rs build [--debug] [--sysroot none|clif|llvm] [--out-dir DIR] [--download-dir DIR] [--no-unstable-features] [--frozen] - ./y.rs test [--debug] [--sysroot none|clif|llvm] [--out-dir DIR] [--download-dir DIR] [--no-unstable-features] [--frozen] - ./y.rs abi-cafe [--debug] [--sysroot none|clif|llvm] [--out-dir DIR] [--download-dir DIR] [--no-unstable-features] [--frozen] - ./y.rs bench [--debug] [--sysroot none|clif|llvm] [--out-dir DIR] [--download-dir DIR] [--no-unstable-features] [--frozen] + ./y.sh prepare [--out-dir DIR] [--download-dir DIR] + ./y.sh build [--debug] [--sysroot none|clif|llvm] [--out-dir DIR] [--download-dir DIR] [--no-unstable-features] [--frozen] + ./y.sh test [--debug] [--sysroot none|clif|llvm] [--out-dir DIR] [--download-dir DIR] [--no-unstable-features] [--frozen] + ./y.sh abi-cafe [--debug] [--sysroot none|clif|llvm] [--out-dir DIR] [--download-dir DIR] [--no-unstable-features] [--frozen] + ./y.sh bench [--debug] [--sysroot none|clif|llvm] [--out-dir DIR] [--download-dir DIR] [--no-unstable-features] [--frozen] OPTIONS: --debug @@ -40,7 +40,7 @@ REQUIREMENTS: * Rustup: By default rustup is used to install the right nightly version. If you don't want to use rustup, you can manually install the nightly version indicated by rust-toolchain.toml and point the CARGO, RUSTC and RUSTDOC env vars to the right executables. - * Git: `./y.rs prepare` uses git for applying patches and on Windows for downloading test repos. - * Curl and tar (non-Windows only): Used by `./y.rs prepare` to download a single commit for + * Git: `./y.sh prepare` uses git for applying patches and on Windows for downloading test repos. + * Curl and tar (non-Windows only): Used by `./y.sh prepare` to download a single commit for repos. Git will be used to clone the whole repo when using Windows. - * [Hyperfine](https://github.com/sharkdp/hyperfine/): Used for benchmarking with `./y.rs bench`. + * [Hyperfine](https://github.com/sharkdp/hyperfine/): Used for benchmarking with `./y.sh bench`. diff --git a/clean_all.sh b/clean_all.sh index cdfc2e143..19405a53d 100755 --- a/clean_all.sh +++ b/clean_all.sh @@ -1,7 +1,7 @@ #!/usr/bin/env bash set -e -rm -rf target/ download/ build/ dist/ y.bin y.bin.dSYM y.exe y.pdb +rm -rf target/ build_system/target download/ build/ dist/ y.bin y.bin.dSYM y.exe y.pdb # Kept for now in case someone updates their checkout of cg_clif before running clean_all.sh # FIXME remove at some point in the future diff --git a/docs/usage.md b/docs/usage.md index 4c2b0fa17..c6210f958 100644 --- a/docs/usage.md +++ b/docs/usage.md @@ -2,7 +2,7 @@ rustc_codegen_cranelift can be used as a near-drop-in replacement for `cargo build` or `cargo run` for existing projects. -Assuming `$cg_clif_dir` is the directory you cloned this repo into and you followed the instructions (`y.rs prepare` and `y.rs build` or `test.sh`). +Assuming `$cg_clif_dir` is the directory you cloned this repo into and you followed the instructions (`y.sh prepare` and `y.sh build` or `test.sh`). ## Cargo diff --git a/scripts/rustup.sh b/scripts/rustup.sh index 3cbeb6375..6b446aafb 100755 --- a/scripts/rustup.sh +++ b/scripts/rustup.sh @@ -32,7 +32,7 @@ case $1 in ./clean_all.sh - ./y.rs prepare + ./y.sh prepare (cd download/sysroot && cargo update && cargo fetch && cp Cargo.lock ../../build_sysroot/) ;; diff --git a/scripts/setup_rust_fork.sh b/scripts/setup_rust_fork.sh index abb09775d..15b16b42b 100644 --- a/scripts/setup_rust_fork.sh +++ b/scripts/setup_rust_fork.sh @@ -1,7 +1,7 @@ #!/usr/bin/env bash set -e -./y.rs build --no-unstable-features +./y.sh build --no-unstable-features echo "[SETUP] Rust fork" git clone https://github.com/rust-lang/rust.git || true diff --git a/test.sh b/test.sh index 13e778453..6357eebf0 100755 --- a/test.sh +++ b/test.sh @@ -1,2 +1,2 @@ #!/usr/bin/env bash -exec ./y.rs test "$@" +exec ./y.sh test "$@" diff --git a/y.rs b/y.rs index a68a10500..e806a64d9 100755 --- a/y.rs +++ b/y.rs @@ -1,35 +1,6 @@ #!/usr/bin/env bash #![deny(unsafe_code)] /*This line is ignored by bash # This block is ignored by rustc -set -e -echo "[BUILD] y.rs" 1>&2 -rustc $0 -o ${0/.rs/.bin} -Cdebuginfo=1 --edition 2021 -exec ${0/.rs/.bin} $@ +echo "Warning: y.rs is a deprecated alias for y.sh" 1>&2 +exec ./y.sh "$@" */ - -#![warn(rust_2018_idioms)] -#![warn(unused_lifetimes)] -#![warn(unreachable_pub)] - -//! The build system for cg_clif -//! -//! # Manual compilation -//! -//! If your system doesn't support shell scripts you can manually compile and run this file using -//! for example: -//! -//! ```shell -//! $ rustc y.rs -o y.bin -//! $ ./y.bin -//! ``` -//! -//! # Naming -//! -//! The name `y.rs` was chosen to not conflict with rustc's `x.py`. - -#[path = "build_system/mod.rs"] -mod build_system; - -fn main() { - build_system::main(); -} diff --git a/y.sh b/y.sh new file mode 100755 index 000000000..bc925a23e --- /dev/null +++ b/y.sh @@ -0,0 +1,6 @@ +#!/usr/bin/env bash + +set -e +echo "[BUILD] build system" 1>&2 +rustc build_system/main.rs -o y.bin -Cdebuginfo=1 --edition 2021 +exec ./y.bin "$@" From 0e4139922e08c306620cf5c43721670c55f3684f Mon Sep 17 00:00:00 2001 From: bjorn3 <17426603+bjorn3@users.noreply.github.com> Date: Fri, 14 Apr 2023 13:17:11 +0000 Subject: [PATCH 03/20] Put patched sources in build/ instead of download/ --- build_system/abi_cafe.rs | 2 +- build_system/build_sysroot.rs | 2 +- build_system/prepare.rs | 33 +++++++++++++++------------------ build_system/tests.rs | 8 ++++---- 4 files changed, 21 insertions(+), 24 deletions(-) diff --git a/build_system/abi_cafe.rs b/build_system/abi_cafe.rs index 9634430d1..ba15db9f9 100644 --- a/build_system/abi_cafe.rs +++ b/build_system/abi_cafe.rs @@ -7,7 +7,7 @@ use super::{CodegenBackend, SysrootKind}; static ABI_CAFE_REPO: GitRepo = GitRepo::github("Gankra", "abi-cafe", "4c6dc8c9c687e2b3a760ff2176ce236872b37212", "abi-cafe"); -static ABI_CAFE: CargoProject = CargoProject::new(&ABI_CAFE_REPO.source_dir(), "abi_cafe"); +static ABI_CAFE: CargoProject = CargoProject::new(&ABI_CAFE_REPO.source_dir(), "abi_cafe_target"); pub(crate) fn run( channel: &str, diff --git a/build_system/build_sysroot.rs b/build_system/build_sysroot.rs index 2ed6272b2..7ceda34bf 100644 --- a/build_system/build_sysroot.rs +++ b/build_system/build_sysroot.rs @@ -156,7 +156,7 @@ impl SysrootTarget { } pub(crate) static ORIG_BUILD_SYSROOT: RelPath = RelPath::SOURCE.join("build_sysroot"); -pub(crate) static BUILD_SYSROOT: RelPath = RelPath::DOWNLOAD.join("sysroot"); +pub(crate) static BUILD_SYSROOT: RelPath = RelPath::BUILD.join("sysroot"); pub(crate) static SYSROOT_RUSTC_VERSION: RelPath = BUILD_SYSROOT.join("rustc_version"); pub(crate) static SYSROOT_SRC: RelPath = BUILD_SYSROOT.join("sysroot_src"); pub(crate) static STANDARD_LIBRARY: CargoProject = diff --git a/build_system/prepare.rs b/build_system/prepare.rs index ac2dc47dd..7bb9eca29 100644 --- a/build_system/prepare.rs +++ b/build_system/prepare.rs @@ -39,9 +39,6 @@ fn prepare_stdlib(dirs: &Dirs, rustc: &Path) { let rustc_version = get_rustc_version(rustc); fs::write(SYSROOT_RUSTC_VERSION.to_path(dirs), &rustc_version).unwrap(); - eprintln!("[GIT] init"); - init_git_repo(&SYSROOT_SRC.to_path(dirs)); - apply_patches(dirs, "stdlib", &SYSROOT_SRC.to_path(dirs)); } @@ -51,15 +48,13 @@ fn prepare_coretests(dirs: &Dirs, rustc: &Path) { eprintln!("[COPY] coretests src"); - fs::create_dir_all(LIBCORE_TESTS_SRC.to_path(dirs)).unwrap(); + // FIXME ensure builds error out or update the copy if any of the files copied here change + LIBCORE_TESTS_SRC.ensure_fresh(dirs); copy_dir_recursively( &sysroot_src_orig.join("library/core/tests"), &LIBCORE_TESTS_SRC.to_path(dirs), ); - eprintln!("[GIT] init"); - init_git_repo(&LIBCORE_TESTS_SRC.to_path(dirs)); - apply_patches(dirs, "coretests", &LIBCORE_TESTS_SRC.to_path(dirs)); } @@ -85,23 +80,23 @@ impl GitRepo { pub(crate) const fn source_dir(&self) -> RelPath { match self.url { - GitRepoUrl::Github { user: _, repo } => RelPath::DOWNLOAD.join(repo), + GitRepoUrl::Github { user: _, repo } => RelPath::BUILD.join(repo), } } pub(crate) fn fetch(&self, dirs: &Dirs) { + let download_dir = match self.url { + GitRepoUrl::Github { user: _, repo } => RelPath::DOWNLOAD.join(repo).to_path(dirs), + }; + let source_dir = self.source_dir(); match self.url { GitRepoUrl::Github { user, repo } => { - clone_repo_shallow_github( - dirs, - &self.source_dir().to_path(dirs), - user, - repo, - self.rev, - ); + clone_repo_shallow_github(dirs, &download_dir, user, repo, self.rev); } } - apply_patches(dirs, self.patch_name, &self.source_dir().to_path(dirs)); + source_dir.ensure_fresh(dirs); + copy_dir_recursively(&download_dir, &source_dir.to_path(dirs)); + apply_patches(dirs, self.patch_name, &source_dir.to_path(dirs)); } } @@ -118,6 +113,8 @@ fn clone_repo(download_dir: &Path, repo: &str, rev: &str) { let mut checkout_cmd = git_command(download_dir, "checkout"); checkout_cmd.arg("-q").arg(rev); spawn_and_wait(checkout_cmd); + + std::fs::remove_dir_all(download_dir.join(".git")).unwrap(); } fn clone_repo_shallow_github(dirs: &Dirs, download_dir: &Path, user: &str, repo: &str, rev: &str) { @@ -165,8 +162,6 @@ fn clone_repo_shallow_github(dirs: &Dirs, download_dir: &Path, user: &str, repo: // Rename unpacked dir to the expected name std::fs::rename(archive_dir, &download_dir).unwrap(); - init_git_repo(&download_dir); - // Cleanup std::fs::remove_file(archive_file).unwrap(); } @@ -206,6 +201,8 @@ fn get_patches(dirs: &Dirs, crate_name: &str) -> Vec { } fn apply_patches(dirs: &Dirs, crate_name: &str, target_dir: &Path) { + init_git_repo(&target_dir); + if crate_name == "" { return; } diff --git a/build_system/tests.rs b/build_system/tests.rs index 7efb96069..f975e43b1 100644 --- a/build_system/tests.rs +++ b/build_system/tests.rs @@ -97,12 +97,12 @@ const BASE_SYSROOT_SUITE: &[TestCase] = &[ pub(crate) static RAND_REPO: GitRepo = GitRepo::github("rust-random", "rand", "50b9a447410860af8d6db9a208c3576886955874", "rand"); -pub(crate) static RAND: CargoProject = CargoProject::new(&RAND_REPO.source_dir(), "rand"); +pub(crate) static RAND: CargoProject = CargoProject::new(&RAND_REPO.source_dir(), "rand_target"); pub(crate) static REGEX_REPO: GitRepo = GitRepo::github("rust-lang", "regex", "32fed9429eafba0ae92a64b01796a0c5a75b88c8", "regex"); -pub(crate) static REGEX: CargoProject = CargoProject::new(®EX_REPO.source_dir(), "regex"); +pub(crate) static REGEX: CargoProject = CargoProject::new(®EX_REPO.source_dir(), "regex_target"); pub(crate) static PORTABLE_SIMD_REPO: GitRepo = GitRepo::github( "rust-lang", @@ -112,9 +112,9 @@ pub(crate) static PORTABLE_SIMD_REPO: GitRepo = GitRepo::github( ); pub(crate) static PORTABLE_SIMD: CargoProject = - CargoProject::new(&PORTABLE_SIMD_REPO.source_dir(), "portable_simd"); + CargoProject::new(&PORTABLE_SIMD_REPO.source_dir(), "portable_simd_target"); -pub(crate) static LIBCORE_TESTS_SRC: RelPath = RelPath::DOWNLOAD.join("coretests_src"); +pub(crate) static LIBCORE_TESTS_SRC: RelPath = RelPath::BUILD.join("coretests_src"); pub(crate) static LIBCORE_TESTS: CargoProject = CargoProject::new(&LIBCORE_TESTS_SRC, "core_tests"); From 75327f8587fad77a25cfe99e52376b6862d75b57 Mon Sep 17 00:00:00 2001 From: bjorn3 <17426603+bjorn3@users.noreply.github.com> Date: Fri, 14 Apr 2023 13:49:41 +0000 Subject: [PATCH 04/20] Reuse existing download in y.sh prepare if fresh --- build_system/abi_cafe.rs | 10 ++++- build_system/bench.rs | 6 +-- build_system/prepare.rs | 85 +++++++++++++++++++++++++++++++++++----- build_system/tests.rs | 19 +++++++-- 4 files changed, 102 insertions(+), 18 deletions(-) diff --git a/build_system/abi_cafe.rs b/build_system/abi_cafe.rs index ba15db9f9..29c127bf5 100644 --- a/build_system/abi_cafe.rs +++ b/build_system/abi_cafe.rs @@ -4,8 +4,13 @@ use super::prepare::GitRepo; use super::utils::{spawn_and_wait, CargoProject, Compiler}; use super::{CodegenBackend, SysrootKind}; -static ABI_CAFE_REPO: GitRepo = - GitRepo::github("Gankra", "abi-cafe", "4c6dc8c9c687e2b3a760ff2176ce236872b37212", "abi-cafe"); +static ABI_CAFE_REPO: GitRepo = GitRepo::github( + "Gankra", + "abi-cafe", + "4c6dc8c9c687e2b3a760ff2176ce236872b37212", + "588df6d66abbe105", + "abi-cafe", +); static ABI_CAFE: CargoProject = CargoProject::new(&ABI_CAFE_REPO.source_dir(), "abi_cafe_target"); @@ -18,6 +23,7 @@ pub(crate) fn run( bootstrap_host_compiler: &Compiler, ) { ABI_CAFE_REPO.fetch(dirs); + ABI_CAFE_REPO.patch(dirs); eprintln!("Building sysroot for abi-cafe"); build_sysroot::build_sysroot( diff --git a/build_system/bench.rs b/build_system/bench.rs index d24803eb7..2bb118000 100644 --- a/build_system/bench.rs +++ b/build_system/bench.rs @@ -10,6 +10,7 @@ static SIMPLE_RAYTRACER_REPO: GitRepo = GitRepo::github( "ebobby", "simple-raytracer", "804a7a21b9e673a482797aa289a18ed480e4d813", + "ad6f59a2331a3f56", "", ); @@ -24,9 +25,8 @@ fn benchmark_simple_raytracer(dirs: &Dirs, bootstrap_host_compiler: &Compiler) { std::process::exit(1); } - if !SIMPLE_RAYTRACER_REPO.source_dir().to_path(dirs).exists() { - SIMPLE_RAYTRACER_REPO.fetch(dirs); - } + SIMPLE_RAYTRACER_REPO.fetch(dirs); + SIMPLE_RAYTRACER_REPO.patch(dirs); let bench_runs = env::var("BENCH_RUNS").unwrap_or_else(|_| "10".to_string()).parse().unwrap(); diff --git a/build_system/prepare.rs b/build_system/prepare.rs index 7bb9eca29..b76164a1d 100644 --- a/build_system/prepare.rs +++ b/build_system/prepare.rs @@ -10,14 +10,18 @@ use super::tests::LIBCORE_TESTS_SRC; use super::utils::{copy_dir_recursively, git_command, retry_spawn_and_wait, spawn_and_wait}; pub(crate) fn prepare(dirs: &Dirs, rustc: &Path) { - RelPath::DOWNLOAD.ensure_fresh(dirs); + RelPath::DOWNLOAD.ensure_exists(dirs); + super::tests::RAND_REPO.fetch(dirs); + super::tests::REGEX_REPO.fetch(dirs); + super::tests::PORTABLE_SIMD_REPO.fetch(dirs); + // FIXME do this on the fly? prepare_stdlib(dirs, rustc); prepare_coretests(dirs, rustc); - super::tests::RAND_REPO.fetch(dirs); - super::tests::REGEX_REPO.fetch(dirs); - super::tests::PORTABLE_SIMD_REPO.fetch(dirs); + super::tests::RAND_REPO.patch(dirs); + super::tests::REGEX_REPO.patch(dirs); + super::tests::PORTABLE_SIMD_REPO.patch(dirs); } fn prepare_stdlib(dirs: &Dirs, rustc: &Path) { @@ -61,6 +65,7 @@ fn prepare_coretests(dirs: &Dirs, rustc: &Path) { pub(crate) struct GitRepo { url: GitRepoUrl, rev: &'static str, + content_hash: &'static str, patch_name: &'static str, } @@ -68,14 +73,49 @@ enum GitRepoUrl { Github { user: &'static str, repo: &'static str }, } +// Note: This uses a hasher which is not cryptographically secure. This is fine as the hash is meant +// to protect against accidental modification and outdated downloads, not against manipulation. +fn hash_file(file: &std::path::Path) -> u64 { + let contents = std::fs::read(file).unwrap(); + #[allow(deprecated)] + let mut hasher = std::hash::SipHasher::new(); + std::hash::Hash::hash(&contents, &mut hasher); + std::hash::Hasher::finish(&hasher) +} + +fn hash_dir(dir: &std::path::Path) -> u64 { + let mut sub_hashes = std::collections::BTreeMap::new(); + for entry in std::fs::read_dir(dir).unwrap() { + let entry = entry.unwrap(); + if entry.file_type().unwrap().is_dir() { + sub_hashes + .insert(entry.file_name().to_str().unwrap().to_owned(), hash_dir(&entry.path())); + } else { + sub_hashes + .insert(entry.file_name().to_str().unwrap().to_owned(), hash_file(&entry.path())); + } + } + #[allow(deprecated)] + let mut hasher = std::hash::SipHasher::new(); + std::hash::Hash::hash(&sub_hashes, &mut hasher); + std::hash::Hasher::finish(&hasher) +} + impl GitRepo { pub(crate) const fn github( user: &'static str, repo: &'static str, rev: &'static str, + content_hash: &'static str, patch_name: &'static str, ) -> GitRepo { - GitRepo { url: GitRepoUrl::Github { user, repo }, rev, patch_name } + GitRepo { url: GitRepoUrl::Github { user, repo }, rev, content_hash, patch_name } + } + + fn download_dir(&self, dirs: &Dirs) -> PathBuf { + match self.url { + GitRepoUrl::Github { user: _, repo } => RelPath::DOWNLOAD.join(repo).to_path(dirs), + } } pub(crate) const fn source_dir(&self) -> RelPath { @@ -85,15 +125,42 @@ impl GitRepo { } pub(crate) fn fetch(&self, dirs: &Dirs) { - let download_dir = match self.url { - GitRepoUrl::Github { user: _, repo } => RelPath::DOWNLOAD.join(repo).to_path(dirs), - }; - let source_dir = self.source_dir(); + let download_dir = self.download_dir(dirs); + + if download_dir.exists() { + let actual_hash = format!("{:016x}", hash_dir(&download_dir)); + if actual_hash == self.content_hash { + println!("[FRESH] {}", download_dir.display()); + return; + } else { + println!( + "Mismatched content hash for {download_dir}: {actual_hash} != {content_hash}. Downloading again.", + download_dir = download_dir.display(), + content_hash = self.content_hash, + ); + } + } + match self.url { GitRepoUrl::Github { user, repo } => { clone_repo_shallow_github(dirs, &download_dir, user, repo, self.rev); } } + + let actual_hash = format!("{:016x}", hash_dir(&download_dir)); + if actual_hash != self.content_hash { + println!( + "Download of {download_dir} failed with mismatched content hash: {actual_hash} != {content_hash}", + download_dir = download_dir.display(), + content_hash = self.content_hash, + ); + std::process::exit(1); + } + } + + pub(crate) fn patch(&self, dirs: &Dirs) { + let download_dir = self.download_dir(dirs); + let source_dir = self.source_dir(); source_dir.ensure_fresh(dirs); copy_dir_recursively(&download_dir, &source_dir.to_path(dirs)); apply_patches(dirs, self.patch_name, &source_dir.to_path(dirs)); diff --git a/build_system/tests.rs b/build_system/tests.rs index f975e43b1..e60f0a4c6 100644 --- a/build_system/tests.rs +++ b/build_system/tests.rs @@ -94,13 +94,23 @@ const BASE_SYSROOT_SUITE: &[TestCase] = &[ // FIXME(rust-random/rand#1293): Newer rand versions fail to test on Windows. Update once this is // fixed. -pub(crate) static RAND_REPO: GitRepo = - GitRepo::github("rust-random", "rand", "50b9a447410860af8d6db9a208c3576886955874", "rand"); +pub(crate) static RAND_REPO: GitRepo = GitRepo::github( + "rust-random", + "rand", + "50b9a447410860af8d6db9a208c3576886955874", + "98b2276210b30e43", + "rand", +); pub(crate) static RAND: CargoProject = CargoProject::new(&RAND_REPO.source_dir(), "rand_target"); -pub(crate) static REGEX_REPO: GitRepo = - GitRepo::github("rust-lang", "regex", "32fed9429eafba0ae92a64b01796a0c5a75b88c8", "regex"); +pub(crate) static REGEX_REPO: GitRepo = GitRepo::github( + "rust-lang", + "regex", + "32fed9429eafba0ae92a64b01796a0c5a75b88c8", + "d6af6507d565aa66", + "regex", +); pub(crate) static REGEX: CargoProject = CargoProject::new(®EX_REPO.source_dir(), "regex_target"); @@ -108,6 +118,7 @@ pub(crate) static PORTABLE_SIMD_REPO: GitRepo = GitRepo::github( "rust-lang", "portable-simd", "ad8afa8c81273b3b49acbea38cd3bcf17a34cf2b", + "1ba291009510070b", "portable-simd", ); From 2c38effe286e1b68a2f1125598ece6c8887d7d49 Mon Sep 17 00:00:00 2001 From: bjorn3 <17426603+bjorn3@users.noreply.github.com> Date: Sun, 28 May 2023 12:37:48 +0000 Subject: [PATCH 05/20] Don't patch in place in apply_patches This will make it easier to skip patching if unnecessary in the future --- build_system/prepare.rs | 42 +++++++++++++++++++++-------------------- 1 file changed, 22 insertions(+), 20 deletions(-) diff --git a/build_system/prepare.rs b/build_system/prepare.rs index b76164a1d..0302946e6 100644 --- a/build_system/prepare.rs +++ b/build_system/prepare.rs @@ -7,7 +7,9 @@ use super::build_sysroot::{BUILD_SYSROOT, ORIG_BUILD_SYSROOT, SYSROOT_RUSTC_VERS use super::path::{Dirs, RelPath}; use super::rustc_info::{get_default_sysroot, get_rustc_version}; use super::tests::LIBCORE_TESTS_SRC; -use super::utils::{copy_dir_recursively, git_command, retry_spawn_and_wait, spawn_and_wait}; +use super::utils::{ + copy_dir_recursively, git_command, remove_dir_if_exists, retry_spawn_and_wait, spawn_and_wait, +}; pub(crate) fn prepare(dirs: &Dirs, rustc: &Path) { RelPath::DOWNLOAD.ensure_exists(dirs); @@ -35,31 +37,24 @@ fn prepare_stdlib(dirs: &Dirs, rustc: &Path) { copy_dir_recursively(&ORIG_BUILD_SYSROOT.to_path(dirs), &BUILD_SYSROOT.to_path(dirs)); fs::create_dir_all(SYSROOT_SRC.to_path(dirs).join("library")).unwrap(); - copy_dir_recursively( - &sysroot_src_orig.join("library"), - &SYSROOT_SRC.to_path(dirs).join("library"), - ); + + apply_patches(dirs, "stdlib", &sysroot_src_orig, &SYSROOT_SRC.to_path(dirs)); let rustc_version = get_rustc_version(rustc); fs::write(SYSROOT_RUSTC_VERSION.to_path(dirs), &rustc_version).unwrap(); - - apply_patches(dirs, "stdlib", &SYSROOT_SRC.to_path(dirs)); } fn prepare_coretests(dirs: &Dirs, rustc: &Path) { let sysroot_src_orig = get_default_sysroot(rustc).join("lib/rustlib/src/rust"); assert!(sysroot_src_orig.exists()); - eprintln!("[COPY] coretests src"); - // FIXME ensure builds error out or update the copy if any of the files copied here change - LIBCORE_TESTS_SRC.ensure_fresh(dirs); - copy_dir_recursively( + apply_patches( + dirs, + "coretests", &sysroot_src_orig.join("library/core/tests"), &LIBCORE_TESTS_SRC.to_path(dirs), ); - - apply_patches(dirs, "coretests", &LIBCORE_TESTS_SRC.to_path(dirs)); } pub(crate) struct GitRepo { @@ -159,11 +154,12 @@ impl GitRepo { } pub(crate) fn patch(&self, dirs: &Dirs) { - let download_dir = self.download_dir(dirs); - let source_dir = self.source_dir(); - source_dir.ensure_fresh(dirs); - copy_dir_recursively(&download_dir, &source_dir.to_path(dirs)); - apply_patches(dirs, self.patch_name, &source_dir.to_path(dirs)); + apply_patches( + dirs, + self.patch_name, + &self.download_dir(dirs), + &self.source_dir().to_path(dirs), + ); } } @@ -267,8 +263,14 @@ fn get_patches(dirs: &Dirs, crate_name: &str) -> Vec { patches } -fn apply_patches(dirs: &Dirs, crate_name: &str, target_dir: &Path) { - init_git_repo(&target_dir); +fn apply_patches(dirs: &Dirs, crate_name: &str, source_dir: &Path, target_dir: &Path) { + // FIXME avoid copy and patch if src, patches and target are unchanged + + remove_dir_if_exists(target_dir); + fs::create_dir_all(target_dir).unwrap(); + copy_dir_recursively(source_dir, target_dir); + + init_git_repo(target_dir); if crate_name == "" { return; From fc23a8a7e0fe32ef18214ced7ee79e111a40df69 Mon Sep 17 00:00:00 2001 From: bjorn3 <17426603+bjorn3@users.noreply.github.com> Date: Sun, 28 May 2023 12:45:05 +0000 Subject: [PATCH 06/20] Lazily patch coretests --- build_system/prepare.rs | 17 +---------------- build_system/tests.rs | 32 ++++++++++++++++++++++++++++---- 2 files changed, 29 insertions(+), 20 deletions(-) diff --git a/build_system/prepare.rs b/build_system/prepare.rs index 0302946e6..8cf51db17 100644 --- a/build_system/prepare.rs +++ b/build_system/prepare.rs @@ -6,7 +6,6 @@ use std::process::Command; use super::build_sysroot::{BUILD_SYSROOT, ORIG_BUILD_SYSROOT, SYSROOT_RUSTC_VERSION, SYSROOT_SRC}; use super::path::{Dirs, RelPath}; use super::rustc_info::{get_default_sysroot, get_rustc_version}; -use super::tests::LIBCORE_TESTS_SRC; use super::utils::{ copy_dir_recursively, git_command, remove_dir_if_exists, retry_spawn_and_wait, spawn_and_wait, }; @@ -19,7 +18,6 @@ pub(crate) fn prepare(dirs: &Dirs, rustc: &Path) { // FIXME do this on the fly? prepare_stdlib(dirs, rustc); - prepare_coretests(dirs, rustc); super::tests::RAND_REPO.patch(dirs); super::tests::REGEX_REPO.patch(dirs); @@ -44,19 +42,6 @@ fn prepare_stdlib(dirs: &Dirs, rustc: &Path) { fs::write(SYSROOT_RUSTC_VERSION.to_path(dirs), &rustc_version).unwrap(); } -fn prepare_coretests(dirs: &Dirs, rustc: &Path) { - let sysroot_src_orig = get_default_sysroot(rustc).join("lib/rustlib/src/rust"); - assert!(sysroot_src_orig.exists()); - - // FIXME ensure builds error out or update the copy if any of the files copied here change - apply_patches( - dirs, - "coretests", - &sysroot_src_orig.join("library/core/tests"), - &LIBCORE_TESTS_SRC.to_path(dirs), - ); -} - pub(crate) struct GitRepo { url: GitRepoUrl, rev: &'static str, @@ -263,7 +248,7 @@ fn get_patches(dirs: &Dirs, crate_name: &str) -> Vec { patches } -fn apply_patches(dirs: &Dirs, crate_name: &str, source_dir: &Path, target_dir: &Path) { +pub(crate) fn apply_patches(dirs: &Dirs, crate_name: &str, source_dir: &Path, target_dir: &Path) { // FIXME avoid copy and patch if src, patches and target are unchanged remove_dir_if_exists(target_dir); diff --git a/build_system/tests.rs b/build_system/tests.rs index e60f0a4c6..ec97be2b2 100644 --- a/build_system/tests.rs +++ b/build_system/tests.rs @@ -1,12 +1,14 @@ use super::build_sysroot; use super::config; use super::path::{Dirs, RelPath}; -use super::prepare::GitRepo; +use super::prepare::{apply_patches, GitRepo}; +use super::rustc_info::get_default_sysroot; use super::utils::{spawn_and_wait, spawn_and_wait_with_input, CargoProject, Compiler}; use super::{CodegenBackend, SysrootKind}; use std::env; use std::ffi::OsStr; use std::fs; +use std::path::PathBuf; use std::process::Command; static BUILD_EXAMPLE_OUT_DIR: RelPath = RelPath::BUILD.join("example"); @@ -125,9 +127,9 @@ pub(crate) static PORTABLE_SIMD_REPO: GitRepo = GitRepo::github( pub(crate) static PORTABLE_SIMD: CargoProject = CargoProject::new(&PORTABLE_SIMD_REPO.source_dir(), "portable_simd_target"); -pub(crate) static LIBCORE_TESTS_SRC: RelPath = RelPath::BUILD.join("coretests_src"); +static LIBCORE_TESTS_SRC: RelPath = RelPath::BUILD.join("coretests_src"); -pub(crate) static LIBCORE_TESTS: CargoProject = CargoProject::new(&LIBCORE_TESTS_SRC, "core_tests"); +static LIBCORE_TESTS: CargoProject = CargoProject::new(&LIBCORE_TESTS_SRC, "core_tests"); const EXTENDED_SYSROOT_SUITE: &[TestCase] = &[ TestCase::custom("test.rust-random/rand", &|runner| { @@ -145,6 +147,13 @@ const EXTENDED_SYSROOT_SUITE: &[TestCase] = &[ } }), TestCase::custom("test.libcore", &|runner| { + apply_patches( + &runner.dirs, + "coretests", + &runner.stdlib_source.join("library/core/tests"), + &LIBCORE_TESTS_SRC.to_path(&runner.dirs), + ); + LIBCORE_TESTS.clean(&runner.dirs); if runner.is_native { @@ -231,6 +240,10 @@ pub(crate) fn run_tests( rustup_toolchain_name: Option<&str>, target_triple: String, ) { + let stdlib_source = + get_default_sysroot(&bootstrap_host_compiler.rustc).join("lib/rustlib/src/rust"); + assert!(stdlib_source.exists()); + if config::get_bool("testsuite.no_sysroot") { let target_compiler = build_sysroot::build_sysroot( dirs, @@ -247,6 +260,7 @@ pub(crate) fn run_tests( target_compiler, use_unstable_features, bootstrap_host_compiler.triple == target_triple, + stdlib_source.clone(), ); BUILD_EXAMPLE_OUT_DIR.ensure_fresh(dirs); @@ -277,6 +291,7 @@ pub(crate) fn run_tests( target_compiler, use_unstable_features, bootstrap_host_compiler.triple == target_triple, + stdlib_source, ); if run_base_sysroot { @@ -299,6 +314,7 @@ struct TestRunner { use_unstable_features: bool, dirs: Dirs, target_compiler: Compiler, + stdlib_source: PathBuf, } impl TestRunner { @@ -307,6 +323,7 @@ impl TestRunner { mut target_compiler: Compiler, use_unstable_features: bool, is_native: bool, + stdlib_source: PathBuf, ) -> Self { if let Ok(rustflags) = env::var("RUSTFLAGS") { target_compiler.rustflags.push(' '); @@ -327,7 +344,14 @@ impl TestRunner { && target_compiler.triple.contains("x86_64") && !target_compiler.triple.contains("windows"); - Self { is_native, jit_supported, use_unstable_features, dirs, target_compiler } + Self { + is_native, + jit_supported, + use_unstable_features, + dirs, + target_compiler, + stdlib_source, + } } fn run_testsuite(&self, tests: &[TestCase]) { From b9129c0d6bb1e2adffb368f5062d4198bce6a7c7 Mon Sep 17 00:00:00 2001 From: bjorn3 <17426603+bjorn3@users.noreply.github.com> Date: Sun, 28 May 2023 13:00:28 +0000 Subject: [PATCH 07/20] Rename a couple of build dirs for consistency --- build_system/tests.rs | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/build_system/tests.rs b/build_system/tests.rs index ec97be2b2..5502e2a23 100644 --- a/build_system/tests.rs +++ b/build_system/tests.rs @@ -125,11 +125,11 @@ pub(crate) static PORTABLE_SIMD_REPO: GitRepo = GitRepo::github( ); pub(crate) static PORTABLE_SIMD: CargoProject = - CargoProject::new(&PORTABLE_SIMD_REPO.source_dir(), "portable_simd_target"); + CargoProject::new(&PORTABLE_SIMD_REPO.source_dir(), "portable-simd_target"); -static LIBCORE_TESTS_SRC: RelPath = RelPath::BUILD.join("coretests_src"); +static LIBCORE_TESTS_SRC: RelPath = RelPath::BUILD.join("coretests"); -static LIBCORE_TESTS: CargoProject = CargoProject::new(&LIBCORE_TESTS_SRC, "core_tests"); +static LIBCORE_TESTS: CargoProject = CargoProject::new(&LIBCORE_TESTS_SRC, "coretests_target"); const EXTENDED_SYSROOT_SUITE: &[TestCase] = &[ TestCase::custom("test.rust-random/rand", &|runner| { From 54eaa5382c8455be4390c7e1e1b9b9f86c5a159d Mon Sep 17 00:00:00 2001 From: bjorn3 <17426603+bjorn3@users.noreply.github.com> Date: Sun, 28 May 2023 17:57:49 +0000 Subject: [PATCH 08/20] Enable compiler-builtins no-asm feature using --features flag --- build_sysroot/Cargo.lock | 1 - build_sysroot/Cargo.toml | 2 -- build_sysroot/src/lib.rs | 2 +- build_system/build_sysroot.rs | 1 + 4 files changed, 2 insertions(+), 4 deletions(-) diff --git a/build_sysroot/Cargo.lock b/build_sysroot/Cargo.lock index ef99acb82..934665a05 100644 --- a/build_sysroot/Cargo.lock +++ b/build_sysroot/Cargo.lock @@ -270,7 +270,6 @@ name = "sysroot" version = "0.0.0" dependencies = [ "alloc", - "compiler_builtins", "core", "proc_macro", "std", diff --git a/build_sysroot/Cargo.toml b/build_sysroot/Cargo.toml index ea9d1c8df..194a5f272 100644 --- a/build_sysroot/Cargo.toml +++ b/build_sysroot/Cargo.toml @@ -9,8 +9,6 @@ std = { path = "./sysroot_src/library/std", features = ["panic_unwind", "backtra test = { path = "./sysroot_src/library/test" } proc_macro = { path = "./sysroot_src/library/proc_macro" } -compiler_builtins = { version = "0.1.87", default-features = false, features = ["no-asm"] } - [patch.crates-io] rustc-std-workspace-core = { path = "./sysroot_src/library/rustc-std-workspace-core" } rustc-std-workspace-alloc = { path = "./sysroot_src/library/rustc-std-workspace-alloc" } diff --git a/build_sysroot/src/lib.rs b/build_sysroot/src/lib.rs index 0c9ac1ac8..8b1378917 100644 --- a/build_sysroot/src/lib.rs +++ b/build_sysroot/src/lib.rs @@ -1 +1 @@ -#![no_std] + diff --git a/build_system/build_sysroot.rs b/build_system/build_sysroot.rs index 7ceda34bf..5c1e30d4e 100644 --- a/build_system/build_sysroot.rs +++ b/build_system/build_sysroot.rs @@ -277,6 +277,7 @@ fn build_clif_sysroot_for_triple( if channel == "release" { build_cmd.arg("--release"); } + build_cmd.arg("--features").arg("std/compiler-builtins-no-asm"); build_cmd.arg("--locked"); build_cmd.env("__CARGO_DEFAULT_LIB_METADATA", "cg_clif"); if compiler.triple.contains("apple") { From 3baee66f9b0cd440d9288ce757cc6caec93eeb67 Mon Sep 17 00:00:00 2001 From: bjorn3 <17426603+bjorn3@users.noreply.github.com> Date: Mon, 29 May 2023 09:58:59 +0000 Subject: [PATCH 09/20] Rework standard library building --- build_sysroot/Cargo.lock | 104 +++++++++++++++++++++++++++++++++- build_sysroot/Cargo.toml | 18 ++---- build_sysroot/src/lib.rs | 1 - build_system/build_sysroot.rs | 11 ++-- build_system/prepare.rs | 13 ++--- 5 files changed, 117 insertions(+), 30 deletions(-) delete mode 100644 build_sysroot/src/lib.rs diff --git a/build_sysroot/Cargo.lock b/build_sysroot/Cargo.lock index 934665a05..1dde9e54d 100644 --- a/build_sysroot/Cargo.lock +++ b/build_sysroot/Cargo.lock @@ -30,8 +30,26 @@ version = "0.0.0" dependencies = [ "compiler_builtins", "core", + "rand", + "rand_xorshift", ] +[[package]] +name = "auxv" +version = "0.3.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e50430f9beb8effb02399fa81c76eeaa26b05e4f03b09285cad8d079c1af5a3d" +dependencies = [ + "byteorder", + "gcc", +] + +[[package]] +name = "byteorder" +version = "1.4.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "14c189c53d098945499cdfa7ecc63567cf3886b3332b312a5b4585d8d3a6a610" + [[package]] name = "cc" version = "1.0.79" @@ -54,12 +72,27 @@ version = "0.1.93" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "76630810d973ecea3dbf611e1b7aecfb1012751ef1ff8de3998f89014a166781" dependencies = [ + "cc", "rustc-std-workspace-core", ] [[package]] name = "core" version = "0.0.0" +dependencies = [ + "rand", + "rand_xorshift", +] + +[[package]] +name = "cupid" +version = "0.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8bad352a84b567cc38a5854e3aa8ee903cb8519a25d0b799b739bafffd1f91a1" +dependencies = [ + "gcc", + "rustc_version", +] [[package]] name = "dlmalloc" @@ -82,6 +115,12 @@ dependencies = [ "rustc-std-workspace-core", ] +[[package]] +name = "gcc" +version = "0.3.55" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8f5f3913fa0bfe7ee1fd8248b6b9f42a5af4b9d65ec2dd2c3c26132b950ecfc2" + [[package]] name = "getopts" version = "0.2.21" @@ -200,6 +239,39 @@ dependencies = [ "std", ] +[[package]] +name = "profiler_builtins" +version = "0.0.0" +dependencies = [ + "cc", + "compiler_builtins", + "core", +] + +[[package]] +name = "rand" +version = "0.8.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "34af8d1a0e25924bc5b7c43c079c942339d8f0a8b57c39049bef581b46327404" +dependencies = [ + "rand_core", +] + +[[package]] +name = "rand_core" +version = "0.6.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ec0be4795e2f6a28069bec0b5ff3e2ac9bafc99e6a9a7dc3547996c5c816922c" + +[[package]] +name = "rand_xorshift" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d25bf25ec5ae4a3f1b92f929810509a2f53d7dca2f50b794ff57e3face536c8f" +dependencies = [ + "rand_core", +] + [[package]] name = "rustc-demangle" version = "0.1.23" @@ -231,6 +303,30 @@ dependencies = [ "std", ] +[[package]] +name = "rustc_version" +version = "0.2.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "138e3e0acb6c9fb258b19b67cb8abd63c00679d2851805ea151465464fe9030a" +dependencies = [ + "semver", +] + +[[package]] +name = "semver" +version = "0.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1d7eb9ef2c18661902cc47e535f9bc51b78acd254da71d375c2f6720d9a40403" +dependencies = [ + "semver-parser", +] + +[[package]] +name = "semver-parser" +version = "0.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "388a1df253eca08550bef6c72392cfe7c30914bf41df5269b68cbd6ff8f570a3" + [[package]] name = "std" version = "0.0.0" @@ -249,6 +345,9 @@ dependencies = [ "object", "panic_abort", "panic_unwind", + "profiler_builtins", + "rand", + "rand_xorshift", "rustc-demangle", "std_detect", "unwind", @@ -259,8 +358,11 @@ dependencies = [ name = "std_detect" version = "0.1.5" dependencies = [ + "auxv", "cfg-if", "compiler_builtins", + "cupid", + "libc", "rustc-std-workspace-alloc", "rustc-std-workspace-core", ] @@ -269,8 +371,6 @@ dependencies = [ name = "sysroot" version = "0.0.0" dependencies = [ - "alloc", - "core", "proc_macro", "std", "test", diff --git a/build_sysroot/Cargo.toml b/build_sysroot/Cargo.toml index 194a5f272..3e5d0c159 100644 --- a/build_sysroot/Cargo.toml +++ b/build_sysroot/Cargo.toml @@ -1,18 +1,10 @@ -[package] -name = "sysroot" -version = "0.0.0" - -[dependencies] -core = { path = "./sysroot_src/library/core" } -alloc = { path = "./sysroot_src/library/alloc" } -std = { path = "./sysroot_src/library/std", features = ["panic_unwind", "backtrace"] } -test = { path = "./sysroot_src/library/test" } -proc_macro = { path = "./sysroot_src/library/proc_macro" } +[workspace] +members = ["./library/sysroot"] [patch.crates-io] -rustc-std-workspace-core = { path = "./sysroot_src/library/rustc-std-workspace-core" } -rustc-std-workspace-alloc = { path = "./sysroot_src/library/rustc-std-workspace-alloc" } -rustc-std-workspace-std = { path = "./sysroot_src/library/rustc-std-workspace-std" } +rustc-std-workspace-core = { path = "./library/rustc-std-workspace-core" } +rustc-std-workspace-alloc = { path = "./library/rustc-std-workspace-alloc" } +rustc-std-workspace-std = { path = "./library/rustc-std-workspace-std" } [profile.dev] lto = "off" diff --git a/build_sysroot/src/lib.rs b/build_sysroot/src/lib.rs deleted file mode 100644 index 8b1378917..000000000 --- a/build_sysroot/src/lib.rs +++ /dev/null @@ -1 +0,0 @@ - diff --git a/build_system/build_sysroot.rs b/build_system/build_sysroot.rs index 5c1e30d4e..09dc43b81 100644 --- a/build_system/build_sysroot.rs +++ b/build_system/build_sysroot.rs @@ -156,11 +156,10 @@ impl SysrootTarget { } pub(crate) static ORIG_BUILD_SYSROOT: RelPath = RelPath::SOURCE.join("build_sysroot"); -pub(crate) static BUILD_SYSROOT: RelPath = RelPath::BUILD.join("sysroot"); -pub(crate) static SYSROOT_RUSTC_VERSION: RelPath = BUILD_SYSROOT.join("rustc_version"); -pub(crate) static SYSROOT_SRC: RelPath = BUILD_SYSROOT.join("sysroot_src"); +pub(crate) static STDLIB_SRC: RelPath = RelPath::BUILD.join("stdlib"); +pub(crate) static SYSROOT_RUSTC_VERSION: RelPath = STDLIB_SRC.join("rustc_version"); pub(crate) static STANDARD_LIBRARY: CargoProject = - CargoProject::new(&BUILD_SYSROOT, "build_sysroot"); + CargoProject::new(&STDLIB_SRC.join("library/sysroot"), "stdlib_target"); pub(crate) static RTSTARTUP_SYSROOT: RelPath = RelPath::BUILD.join("rtstartup"); #[must_use] @@ -277,7 +276,7 @@ fn build_clif_sysroot_for_triple( if channel == "release" { build_cmd.arg("--release"); } - build_cmd.arg("--features").arg("std/compiler-builtins-no-asm"); + build_cmd.arg("--features").arg("compiler-builtins-no-asm backtrace panic-unwind"); build_cmd.arg("--locked"); build_cmd.env("__CARGO_DEFAULT_LIB_METADATA", "cg_clif"); if compiler.triple.contains("apple") { @@ -307,7 +306,7 @@ fn build_rtstartup(dirs: &Dirs, compiler: &Compiler) -> Option { RTSTARTUP_SYSROOT.ensure_fresh(dirs); - let rtstartup_src = SYSROOT_SRC.to_path(dirs).join("library").join("rtstartup"); + let rtstartup_src = STDLIB_SRC.to_path(dirs).join("library").join("rtstartup"); let mut target_libs = SysrootTarget { triple: compiler.triple.clone(), libs: vec![] }; for file in ["rsbegin", "rsend"] { diff --git a/build_system/prepare.rs b/build_system/prepare.rs index 8cf51db17..bc249f4cd 100644 --- a/build_system/prepare.rs +++ b/build_system/prepare.rs @@ -3,7 +3,7 @@ use std::fs; use std::path::{Path, PathBuf}; use std::process::Command; -use super::build_sysroot::{BUILD_SYSROOT, ORIG_BUILD_SYSROOT, SYSROOT_RUSTC_VERSION, SYSROOT_SRC}; +use super::build_sysroot::{ORIG_BUILD_SYSROOT, STDLIB_SRC, SYSROOT_RUSTC_VERSION}; use super::path::{Dirs, RelPath}; use super::rustc_info::{get_default_sysroot, get_rustc_version}; use super::utils::{ @@ -28,15 +28,10 @@ fn prepare_stdlib(dirs: &Dirs, rustc: &Path) { let sysroot_src_orig = get_default_sysroot(rustc).join("lib/rustlib/src/rust"); assert!(sysroot_src_orig.exists()); - eprintln!("[COPY] stdlib src"); + apply_patches(dirs, "stdlib", &sysroot_src_orig, &STDLIB_SRC.to_path(dirs)); // FIXME ensure builds error out or update the copy if any of the files copied here change - BUILD_SYSROOT.ensure_fresh(dirs); - copy_dir_recursively(&ORIG_BUILD_SYSROOT.to_path(dirs), &BUILD_SYSROOT.to_path(dirs)); - - fs::create_dir_all(SYSROOT_SRC.to_path(dirs).join("library")).unwrap(); - - apply_patches(dirs, "stdlib", &sysroot_src_orig, &SYSROOT_SRC.to_path(dirs)); + copy_dir_recursively(&ORIG_BUILD_SYSROOT.to_path(dirs), &STDLIB_SRC.to_path(dirs)); let rustc_version = get_rustc_version(rustc); fs::write(SYSROOT_RUSTC_VERSION.to_path(dirs), &rustc_version).unwrap(); @@ -251,6 +246,8 @@ fn get_patches(dirs: &Dirs, crate_name: &str) -> Vec { pub(crate) fn apply_patches(dirs: &Dirs, crate_name: &str, source_dir: &Path, target_dir: &Path) { // FIXME avoid copy and patch if src, patches and target are unchanged + eprintln!("[COPY] {crate_name} source"); + remove_dir_if_exists(target_dir); fs::create_dir_all(target_dir).unwrap(); copy_dir_recursively(source_dir, target_dir); From 67f9fe6863cede6ca5f41b938e4f3188feb24c65 Mon Sep 17 00:00:00 2001 From: bjorn3 <17426603+bjorn3@users.noreply.github.com> Date: Mon, 29 May 2023 11:51:37 +0000 Subject: [PATCH 10/20] Lazily patch all test projects --- build_system/prepare.rs | 4 ---- build_system/tests.rs | 8 ++++++++ 2 files changed, 8 insertions(+), 4 deletions(-) diff --git a/build_system/prepare.rs b/build_system/prepare.rs index bc249f4cd..115575bff 100644 --- a/build_system/prepare.rs +++ b/build_system/prepare.rs @@ -18,10 +18,6 @@ pub(crate) fn prepare(dirs: &Dirs, rustc: &Path) { // FIXME do this on the fly? prepare_stdlib(dirs, rustc); - - super::tests::RAND_REPO.patch(dirs); - super::tests::REGEX_REPO.patch(dirs); - super::tests::PORTABLE_SIMD_REPO.patch(dirs); } fn prepare_stdlib(dirs: &Dirs, rustc: &Path) { diff --git a/build_system/tests.rs b/build_system/tests.rs index 5502e2a23..d104efd61 100644 --- a/build_system/tests.rs +++ b/build_system/tests.rs @@ -133,6 +133,8 @@ static LIBCORE_TESTS: CargoProject = CargoProject::new(&LIBCORE_TESTS_SRC, "core const EXTENDED_SYSROOT_SUITE: &[TestCase] = &[ TestCase::custom("test.rust-random/rand", &|runner| { + RAND_REPO.patch(&runner.dirs); + RAND.clean(&runner.dirs); if runner.is_native { @@ -168,6 +170,8 @@ const EXTENDED_SYSROOT_SUITE: &[TestCase] = &[ } }), TestCase::custom("test.regex-shootout-regex-dna", &|runner| { + REGEX_REPO.patch(&runner.dirs); + REGEX.clean(&runner.dirs); let mut build_cmd = REGEX.build(&runner.target_compiler, &runner.dirs); @@ -200,6 +204,8 @@ const EXTENDED_SYSROOT_SUITE: &[TestCase] = &[ } }), TestCase::custom("test.regex", &|runner| { + REGEX_REPO.patch(&runner.dirs); + REGEX.clean(&runner.dirs); if runner.is_native { @@ -216,6 +222,8 @@ const EXTENDED_SYSROOT_SUITE: &[TestCase] = &[ } }), TestCase::custom("test.portable-simd", &|runner| { + PORTABLE_SIMD_REPO.patch(&runner.dirs); + PORTABLE_SIMD.clean(&runner.dirs); let mut build_cmd = PORTABLE_SIMD.build(&runner.target_compiler, &runner.dirs); From d3da972441f6efd16c1b1aa5c2672cc218949bdb Mon Sep 17 00:00:00 2001 From: bjorn3 <17426603+bjorn3@users.noreply.github.com> Date: Mon, 29 May 2023 12:19:38 +0000 Subject: [PATCH 11/20] Write stdlib workspace Cargo.toml directly in prepare.rs --- build_sysroot/Cargo.toml | 26 -------------------------- build_system/prepare.rs | 33 +++++++++++++++++++++++++++++++++ 2 files changed, 33 insertions(+), 26 deletions(-) delete mode 100644 build_sysroot/Cargo.toml diff --git a/build_sysroot/Cargo.toml b/build_sysroot/Cargo.toml deleted file mode 100644 index 3e5d0c159..000000000 --- a/build_sysroot/Cargo.toml +++ /dev/null @@ -1,26 +0,0 @@ -[workspace] -members = ["./library/sysroot"] - -[patch.crates-io] -rustc-std-workspace-core = { path = "./library/rustc-std-workspace-core" } -rustc-std-workspace-alloc = { path = "./library/rustc-std-workspace-alloc" } -rustc-std-workspace-std = { path = "./library/rustc-std-workspace-std" } - -[profile.dev] -lto = "off" - -[profile.release] -debug = true -incremental = true -lto = "off" - -# Mandatory for correctly compiling compiler-builtins -[profile.dev.package.compiler_builtins] -debug-assertions = false -overflow-checks = false -codegen-units = 10000 - -[profile.release.package.compiler_builtins] -debug-assertions = false -overflow-checks = false -codegen-units = 10000 diff --git a/build_system/prepare.rs b/build_system/prepare.rs index 115575bff..4f26cb5e8 100644 --- a/build_system/prepare.rs +++ b/build_system/prepare.rs @@ -29,6 +29,39 @@ fn prepare_stdlib(dirs: &Dirs, rustc: &Path) { // FIXME ensure builds error out or update the copy if any of the files copied here change copy_dir_recursively(&ORIG_BUILD_SYSROOT.to_path(dirs), &STDLIB_SRC.to_path(dirs)); + std::fs::write( + STDLIB_SRC.to_path(dirs).join("Cargo.toml"), + r#" +[workspace] +members = ["./library/sysroot"] + +[patch.crates-io] +rustc-std-workspace-core = { path = "./library/rustc-std-workspace-core" } +rustc-std-workspace-alloc = { path = "./library/rustc-std-workspace-alloc" } +rustc-std-workspace-std = { path = "./library/rustc-std-workspace-std" } + +[profile.dev] +lto = "off" + +[profile.release] +debug = true +incremental = true +lto = "off" + +# Mandatory for correctly compiling compiler-builtins +[profile.dev.package.compiler_builtins] +debug-assertions = false +overflow-checks = false +codegen-units = 10000 + +[profile.release.package.compiler_builtins] +debug-assertions = false +overflow-checks = false +codegen-units = 10000 +"#, + ) + .unwrap(); + let rustc_version = get_rustc_version(rustc); fs::write(SYSROOT_RUSTC_VERSION.to_path(dirs), &rustc_version).unwrap(); } From 8ad9e9f861c9cbdd85c99df73a4a2d03a0d15014 Mon Sep 17 00:00:00 2001 From: bjorn3 <17426603+bjorn3@users.noreply.github.com> Date: Mon, 29 May 2023 12:09:40 +0000 Subject: [PATCH 12/20] Ensure everything has a lockfile --- build_system/build_sysroot.rs | 2 - build_system/prepare.rs | 13 +- build_system/utils.rs | 3 +- patches/coretests-lock.toml | 35 ++ patches/portable-simd-lock.toml | 304 ++++++++++++ patches/rand-lock.toml | 346 ++++++++++++++ patches/regex-lock.toml | 439 ++++++++++++++++++ .../Cargo.lock => patches/stdlib-lock.toml | 0 scripts/rustup.sh | 4 +- 9 files changed, 1136 insertions(+), 10 deletions(-) create mode 100644 patches/coretests-lock.toml create mode 100644 patches/portable-simd-lock.toml create mode 100644 patches/rand-lock.toml create mode 100644 patches/regex-lock.toml rename build_sysroot/Cargo.lock => patches/stdlib-lock.toml (100%) diff --git a/build_system/build_sysroot.rs b/build_system/build_sysroot.rs index 09dc43b81..1045023c3 100644 --- a/build_system/build_sysroot.rs +++ b/build_system/build_sysroot.rs @@ -155,7 +155,6 @@ impl SysrootTarget { } } -pub(crate) static ORIG_BUILD_SYSROOT: RelPath = RelPath::SOURCE.join("build_sysroot"); pub(crate) static STDLIB_SRC: RelPath = RelPath::BUILD.join("stdlib"); pub(crate) static SYSROOT_RUSTC_VERSION: RelPath = STDLIB_SRC.join("rustc_version"); pub(crate) static STANDARD_LIBRARY: CargoProject = @@ -277,7 +276,6 @@ fn build_clif_sysroot_for_triple( build_cmd.arg("--release"); } build_cmd.arg("--features").arg("compiler-builtins-no-asm backtrace panic-unwind"); - build_cmd.arg("--locked"); build_cmd.env("__CARGO_DEFAULT_LIB_METADATA", "cg_clif"); if compiler.triple.contains("apple") { build_cmd.env("CARGO_PROFILE_RELEASE_SPLIT_DEBUGINFO", "packed"); diff --git a/build_system/prepare.rs b/build_system/prepare.rs index 4f26cb5e8..c6bc796f3 100644 --- a/build_system/prepare.rs +++ b/build_system/prepare.rs @@ -3,7 +3,7 @@ use std::fs; use std::path::{Path, PathBuf}; use std::process::Command; -use super::build_sysroot::{ORIG_BUILD_SYSROOT, STDLIB_SRC, SYSROOT_RUSTC_VERSION}; +use super::build_sysroot::{STDLIB_SRC, SYSROOT_RUSTC_VERSION}; use super::path::{Dirs, RelPath}; use super::rustc_info::{get_default_sysroot, get_rustc_version}; use super::utils::{ @@ -26,9 +26,6 @@ fn prepare_stdlib(dirs: &Dirs, rustc: &Path) { apply_patches(dirs, "stdlib", &sysroot_src_orig, &STDLIB_SRC.to_path(dirs)); - // FIXME ensure builds error out or update the copy if any of the files copied here change - copy_dir_recursively(&ORIG_BUILD_SYSROOT.to_path(dirs), &STDLIB_SRC.to_path(dirs)); - std::fs::write( STDLIB_SRC.to_path(dirs).join("Cargo.toml"), r#" @@ -297,4 +294,12 @@ pub(crate) fn apply_patches(dirs: &Dirs, crate_name: &str, source_dir: &Path, ta apply_patch_cmd.arg(patch).arg("-q"); spawn_and_wait(apply_patch_cmd); } + + let source_lockfile = RelPath::PATCHES.to_path(dirs).join(format!("{crate_name}-lock.toml")); + let target_lockfile = target_dir.join("Cargo.lock"); + if source_lockfile.exists() { + fs::copy(source_lockfile, target_lockfile).unwrap(); + } else { + assert!(target_lockfile.exists()); + } } diff --git a/build_system/utils.rs b/build_system/utils.rs index 3e12ed22e..e7fcb2644 100644 --- a/build_system/utils.rs +++ b/build_system/utils.rs @@ -81,7 +81,8 @@ impl CargoProject { .arg("--manifest-path") .arg(self.manifest_path(dirs)) .arg("--target-dir") - .arg(self.target_dir(dirs)); + .arg(self.target_dir(dirs)) + .arg("--locked"); if dirs.frozen { cmd.arg("--frozen"); diff --git a/patches/coretests-lock.toml b/patches/coretests-lock.toml new file mode 100644 index 000000000..af8f28a19 --- /dev/null +++ b/patches/coretests-lock.toml @@ -0,0 +1,35 @@ +# This file is automatically @generated by Cargo. +# It is not intended for manual editing. +version = 3 + +[[package]] +name = "coretests" +version = "0.0.0" +dependencies = [ + "rand", + "rand_xorshift", +] + +[[package]] +name = "rand" +version = "0.8.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "34af8d1a0e25924bc5b7c43c079c942339d8f0a8b57c39049bef581b46327404" +dependencies = [ + "rand_core", +] + +[[package]] +name = "rand_core" +version = "0.6.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ec0be4795e2f6a28069bec0b5ff3e2ac9bafc99e6a9a7dc3547996c5c816922c" + +[[package]] +name = "rand_xorshift" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d25bf25ec5ae4a3f1b92f929810509a2f53d7dca2f50b794ff57e3face536c8f" +dependencies = [ + "rand_core", +] diff --git a/patches/portable-simd-lock.toml b/patches/portable-simd-lock.toml new file mode 100644 index 000000000..e7db1fd2c --- /dev/null +++ b/patches/portable-simd-lock.toml @@ -0,0 +1,304 @@ +# This file is automatically @generated by Cargo. +# It is not intended for manual editing. +version = 3 + +[[package]] +name = "autocfg" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d468802bab17cbc0cc575e9b053f41e72aa36bfa6b7f55e3529ffa43161b97fa" + +[[package]] +name = "bitflags" +version = "1.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bef38d45163c2f1dde094a7dfd33ccf595c92905c8f8f4fdc18d06fb1037718a" + +[[package]] +name = "bumpalo" +version = "3.13.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a3e2c3daef883ecc1b5d58c15adae93470a91d425f3532ba1695849656af3fc1" + +[[package]] +name = "byteorder" +version = "1.4.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "14c189c53d098945499cdfa7ecc63567cf3886b3332b312a5b4585d8d3a6a610" + +[[package]] +name = "cfg-if" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "baf1de4339761588bc0619e3cbc0120ee582ebb74b53b4efbf79117bd2da40fd" + +[[package]] +name = "console_error_panic_hook" +version = "0.1.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a06aeb73f470f66dcdbf7223caeebb85984942f22f1adb2a088cf9668146bbbc" +dependencies = [ + "cfg-if", + "wasm-bindgen", +] + +[[package]] +name = "core_simd" +version = "0.1.0" +dependencies = [ + "proptest", + "std_float", + "test_helpers", + "wasm-bindgen", + "wasm-bindgen-test", +] + +[[package]] +name = "js-sys" +version = "0.3.63" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2f37a4a5928311ac501dee68b3c7613a1037d0edb30c8e5427bd832d55d1b790" +dependencies = [ + "wasm-bindgen", +] + +[[package]] +name = "log" +version = "0.4.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "518ef76f2f87365916b142844c16d8fefd85039bc5699050210a7778ee1cd1de" + +[[package]] +name = "num-traits" +version = "0.2.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "578ede34cf02f8924ab9447f50c28075b4d3e5b269972345e7e0372b38c6cdcd" +dependencies = [ + "autocfg", +] + +[[package]] +name = "once_cell" +version = "1.17.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9670a07f94779e00908f3e686eab508878ebb390ba6e604d3a284c00e8d0487b" + +[[package]] +name = "ppv-lite86" +version = "0.2.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5b40af805b3121feab8a3c29f04d8ad262fa8e0561883e7653e024ae4479e6de" + +[[package]] +name = "proc-macro2" +version = "1.0.59" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6aeca18b86b413c660b781aa319e4e2648a3e6f9eadc9b47e9038e6fe9f3451b" +dependencies = [ + "unicode-ident", +] + +[[package]] +name = "proptest" +version = "0.10.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "12e6c80c1139113c28ee4670dc50cc42915228b51f56a9e407f0ec60f966646f" +dependencies = [ + "bitflags", + "byteorder", + "num-traits", + "rand", + "rand_chacha", + "rand_xorshift", +] + +[[package]] +name = "quote" +version = "1.0.28" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1b9ab9c7eadfd8df19006f1cf1a4aed13540ed5cbc047010ece5826e10825488" +dependencies = [ + "proc-macro2", +] + +[[package]] +name = "rand" +version = "0.7.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6a6b1679d49b24bbfe0c803429aa1874472f50d9b363131f0e89fc356b544d03" +dependencies = [ + "rand_chacha", + "rand_core", + "rand_hc", +] + +[[package]] +name = "rand_chacha" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f4c8ed856279c9737206bf725bf36935d8666ead7aa69b52be55af369d193402" +dependencies = [ + "ppv-lite86", + "rand_core", +] + +[[package]] +name = "rand_core" +version = "0.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "90bde5296fc891b0cef12a6d03ddccc162ce7b2aff54160af9338f8d40df6d19" + +[[package]] +name = "rand_hc" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ca3129af7b92a17112d59ad498c6f81eaf463253766b90396d39ea7a39d6613c" +dependencies = [ + "rand_core", +] + +[[package]] +name = "rand_xorshift" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "77d416b86801d23dde1aa643023b775c3a462efc0ed96443add11546cdf1dca8" +dependencies = [ + "rand_core", +] + +[[package]] +name = "scoped-tls" +version = "1.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e1cf6437eb19a8f4a6cc0f7dca544973b0b78843adbfeb3683d1a94a0024a294" + +[[package]] +name = "std_float" +version = "0.1.0" +dependencies = [ + "core_simd", +] + +[[package]] +name = "syn" +version = "2.0.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "32d41677bcbe24c20c52e7c70b0d8db04134c5d1066bf98662e2871ad200ea3e" +dependencies = [ + "proc-macro2", + "quote", + "unicode-ident", +] + +[[package]] +name = "test_helpers" +version = "0.1.0" +dependencies = [ + "proptest", +] + +[[package]] +name = "unicode-ident" +version = "1.0.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b15811caf2415fb889178633e7724bad2509101cde276048e013b9def5e51fa0" + +[[package]] +name = "wasm-bindgen" +version = "0.2.86" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5bba0e8cb82ba49ff4e229459ff22a191bbe9a1cb3a341610c9c33efc27ddf73" +dependencies = [ + "cfg-if", + "wasm-bindgen-macro", +] + +[[package]] +name = "wasm-bindgen-backend" +version = "0.2.86" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "19b04bc93f9d6bdee709f6bd2118f57dd6679cf1176a1af464fca3ab0d66d8fb" +dependencies = [ + "bumpalo", + "log", + "once_cell", + "proc-macro2", + "quote", + "syn", + "wasm-bindgen-shared", +] + +[[package]] +name = "wasm-bindgen-futures" +version = "0.4.36" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2d1985d03709c53167ce907ff394f5316aa22cb4e12761295c5dc57dacb6297e" +dependencies = [ + "cfg-if", + "js-sys", + "wasm-bindgen", + "web-sys", +] + +[[package]] +name = "wasm-bindgen-macro" +version = "0.2.86" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "14d6b024f1a526bb0234f52840389927257beb670610081360e5a03c5df9c258" +dependencies = [ + "quote", + "wasm-bindgen-macro-support", +] + +[[package]] +name = "wasm-bindgen-macro-support" +version = "0.2.86" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e128beba882dd1eb6200e1dc92ae6c5dbaa4311aa7bb211ca035779e5efc39f8" +dependencies = [ + "proc-macro2", + "quote", + "syn", + "wasm-bindgen-backend", + "wasm-bindgen-shared", +] + +[[package]] +name = "wasm-bindgen-shared" +version = "0.2.86" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ed9d5b4305409d1fc9482fee2d7f9bcbf24b3972bf59817ef757e23982242a93" + +[[package]] +name = "wasm-bindgen-test" +version = "0.3.36" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c9e636f3a428ff62b3742ebc3c70e254dfe12b8c2b469d688ea59cdd4abcf502" +dependencies = [ + "console_error_panic_hook", + "js-sys", + "scoped-tls", + "wasm-bindgen", + "wasm-bindgen-futures", + "wasm-bindgen-test-macro", +] + +[[package]] +name = "wasm-bindgen-test-macro" +version = "0.3.36" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f18c1fad2f7c4958e7bcce014fa212f59a65d5e3721d0f77e6c0b27ede936ba3" +dependencies = [ + "proc-macro2", + "quote", +] + +[[package]] +name = "web-sys" +version = "0.3.63" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3bdd9ef4e984da1187bf8110c5cf5b845fbc87a23602cdf912386a76fcd3a7c2" +dependencies = [ + "js-sys", + "wasm-bindgen", +] diff --git a/patches/rand-lock.toml b/patches/rand-lock.toml new file mode 100644 index 000000000..66c515731 --- /dev/null +++ b/patches/rand-lock.toml @@ -0,0 +1,346 @@ +# This file is automatically @generated by Cargo. +# It is not intended for manual editing. +version = 3 + +[[package]] +name = "autocfg" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d468802bab17cbc0cc575e9b053f41e72aa36bfa6b7f55e3529ffa43161b97fa" + +[[package]] +name = "average" +version = "0.13.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "843ec791d3f24503bbf72bbd5e49a3ab4dbb4bcd0a8ef6b0c908efa73caa27b1" +dependencies = [ + "easy-cast", + "float-ord", + "num-traits", +] + +[[package]] +name = "bincode" +version = "1.3.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b1f45e9417d87227c7a56d22e471c6206462cba514c7590c09aff4cf6d1ddcad" +dependencies = [ + "serde", +] + +[[package]] +name = "cfg-if" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "baf1de4339761588bc0619e3cbc0120ee582ebb74b53b4efbf79117bd2da40fd" + +[[package]] +name = "crossbeam-channel" +version = "0.5.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a33c2bf77f2df06183c3aa30d1e96c0695a313d4f9c453cc3762a6db39f99200" +dependencies = [ + "cfg-if", + "crossbeam-utils", +] + +[[package]] +name = "crossbeam-deque" +version = "0.8.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ce6fd6f855243022dcecf8702fef0c297d4338e226845fe067f6341ad9fa0cef" +dependencies = [ + "cfg-if", + "crossbeam-epoch", + "crossbeam-utils", +] + +[[package]] +name = "crossbeam-epoch" +version = "0.9.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "46bd5f3f85273295a9d14aedfb86f6aadbff6d8f5295c4a9edb08e819dcf5695" +dependencies = [ + "autocfg", + "cfg-if", + "crossbeam-utils", + "memoffset", + "scopeguard", +] + +[[package]] +name = "crossbeam-utils" +version = "0.8.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3c063cd8cc95f5c377ed0d4b49a4b21f632396ff690e8470c29b3359b346984b" +dependencies = [ + "cfg-if", +] + +[[package]] +name = "easy-cast" +version = "0.4.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4bd102ee8c418348759919b83b81cdbdc933ffe29740b903df448b4bafaa348e" +dependencies = [ + "libm", +] + +[[package]] +name = "either" +version = "1.8.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7fcaabb2fef8c910e7f4c7ce9f67a1283a1715879a7c230ca9d6d1ae31f16d91" + +[[package]] +name = "float-ord" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8ce81f49ae8a0482e4c55ea62ebbd7e5a686af544c00b9d090bba3ff9be97b3d" + +[[package]] +name = "getrandom" +version = "0.2.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c85e1d9ab2eadba7e5040d4e09cbd6d072b76a557ad64e797c2cb9d4da21d7e4" +dependencies = [ + "cfg-if", + "libc", + "wasi", +] + +[[package]] +name = "hermit-abi" +version = "0.2.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ee512640fe35acbfb4bb779db6f0d80704c2cacfa2e39b601ef3e3f47d1ae4c7" +dependencies = [ + "libc", +] + +[[package]] +name = "itoa" +version = "1.0.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "453ad9f582a441959e5f0d088b02ce04cfe8d51a8eaf077f12ac6d3e94164ca6" + +[[package]] +name = "libc" +version = "0.2.144" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2b00cc1c228a6782d0f076e7b232802e0c5689d41bb5df366f2a6b6621cfdfe1" + +[[package]] +name = "libm" +version = "0.2.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f7012b1bbb0719e1097c47611d3898568c546d597c2e74d66f6087edd5233ff4" + +[[package]] +name = "log" +version = "0.4.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "518ef76f2f87365916b142844c16d8fefd85039bc5699050210a7778ee1cd1de" + +[[package]] +name = "memoffset" +version = "0.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d61c719bcfbcf5d62b3a09efa6088de8c54bc0bfcd3ea7ae39fcc186108b8de1" +dependencies = [ + "autocfg", +] + +[[package]] +name = "num-traits" +version = "0.2.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "578ede34cf02f8924ab9447f50c28075b4d3e5b269972345e7e0372b38c6cdcd" +dependencies = [ + "autocfg", + "libm", +] + +[[package]] +name = "num_cpus" +version = "1.15.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0fac9e2da13b5eb447a6ce3d392f23a29d8694bff781bf03a16cd9ac8697593b" +dependencies = [ + "hermit-abi", + "libc", +] + +[[package]] +name = "ppv-lite86" +version = "0.2.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5b40af805b3121feab8a3c29f04d8ad262fa8e0561883e7653e024ae4479e6de" + +[[package]] +name = "proc-macro2" +version = "1.0.59" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6aeca18b86b413c660b781aa319e4e2648a3e6f9eadc9b47e9038e6fe9f3451b" +dependencies = [ + "unicode-ident", +] + +[[package]] +name = "quote" +version = "1.0.28" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1b9ab9c7eadfd8df19006f1cf1a4aed13540ed5cbc047010ece5826e10825488" +dependencies = [ + "proc-macro2", +] + +[[package]] +name = "rand" +version = "0.9.0" +dependencies = [ + "bincode", + "libc", + "log", + "rand_chacha", + "rand_core", + "rand_pcg", + "rayon", + "serde", +] + +[[package]] +name = "rand_chacha" +version = "0.4.0" +dependencies = [ + "ppv-lite86", + "rand_core", + "serde", + "serde_json", +] + +[[package]] +name = "rand_core" +version = "0.7.0" +dependencies = [ + "getrandom", + "serde", +] + +[[package]] +name = "rand_distr" +version = "0.5.0" +dependencies = [ + "average", + "num-traits", + "rand", + "rand_pcg", + "serde", + "special", +] + +[[package]] +name = "rand_pcg" +version = "0.4.0" +dependencies = [ + "bincode", + "rand_core", + "serde", +] + +[[package]] +name = "rayon" +version = "1.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1d2df5196e37bcc87abebc0053e20787d73847bb33134a69841207dd0a47f03b" +dependencies = [ + "either", + "rayon-core", +] + +[[package]] +name = "rayon-core" +version = "1.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4b8f95bd6966f5c87776639160a66bd8ab9895d9d4ab01ddba9fc60661aebe8d" +dependencies = [ + "crossbeam-channel", + "crossbeam-deque", + "crossbeam-utils", + "num_cpus", +] + +[[package]] +name = "ryu" +version = "1.0.13" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f91339c0467de62360649f8d3e185ca8de4224ff281f66000de5eb2a77a79041" + +[[package]] +name = "scopeguard" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d29ab0c6d3fc0ee92fe66e2d99f700eab17a8d57d1c1d3b748380fb20baa78cd" + +[[package]] +name = "serde" +version = "1.0.163" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2113ab51b87a539ae008b5c6c02dc020ffa39afd2d83cffcb3f4eb2722cebec2" +dependencies = [ + "serde_derive", +] + +[[package]] +name = "serde_derive" +version = "1.0.163" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8c805777e3930c8883389c602315a24224bcc738b63905ef87cd1420353ea93e" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "serde_json" +version = "1.0.96" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "057d394a50403bcac12672b2b18fb387ab6d289d957dab67dd201875391e52f1" +dependencies = [ + "itoa", + "ryu", + "serde", +] + +[[package]] +name = "special" +version = "0.8.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "24a65e074159b75dcf173a4733ab2188baac24967b5c8ec9ed87ae15fcbc7636" +dependencies = [ + "libc", +] + +[[package]] +name = "syn" +version = "2.0.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "32d41677bcbe24c20c52e7c70b0d8db04134c5d1066bf98662e2871ad200ea3e" +dependencies = [ + "proc-macro2", + "quote", + "unicode-ident", +] + +[[package]] +name = "unicode-ident" +version = "1.0.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b15811caf2415fb889178633e7724bad2509101cde276048e013b9def5e51fa0" + +[[package]] +name = "wasi" +version = "0.11.0+wasi-snapshot-preview1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9c8d87e72b64a3b4db28d11ce29237c246188f4f51057d65a7eab63b7987e423" diff --git a/patches/regex-lock.toml b/patches/regex-lock.toml new file mode 100644 index 000000000..0e4a33b90 --- /dev/null +++ b/patches/regex-lock.toml @@ -0,0 +1,439 @@ +# This file is automatically @generated by Cargo. +# It is not intended for manual editing. +version = 3 + +[[package]] +name = "aho-corasick" +version = "0.7.20" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cc936419f96fa211c1b9166887b38e5e40b19958e5b895be7c1f93adec7071ac" +dependencies = [ + "memchr", +] + +[[package]] +name = "bitflags" +version = "1.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bef38d45163c2f1dde094a7dfd33ccf595c92905c8f8f4fdc18d06fb1037718a" + +[[package]] +name = "bzip2" +version = "0.3.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "42b7c3cbf0fa9c1b82308d57191728ca0256cb821220f4e2fd410a72ade26e3b" +dependencies = [ + "bzip2-sys", + "libc", +] + +[[package]] +name = "bzip2-sys" +version = "0.1.11+1.0.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "736a955f3fa7875102d57c82b8cac37ec45224a07fd32d58f9f7a186b6cd4cdc" +dependencies = [ + "cc", + "libc", + "pkg-config", +] + +[[package]] +name = "cc" +version = "1.0.79" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "50d30906286121d95be3d479533b458f87493b30a4b5f79a607db8f5d11aa91f" + +[[package]] +name = "cfg-if" +version = "0.1.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4785bdd1c96b2a846b2bd7cc02e86b6b3dbf14e7e53446c4f54c92a361040822" + +[[package]] +name = "cfg-if" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "baf1de4339761588bc0619e3cbc0120ee582ebb74b53b4efbf79117bd2da40fd" + +[[package]] +name = "docopt" +version = "1.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7f3f119846c823f9eafcf953a8f6ffb6ed69bf6240883261a7f13b634579a51f" +dependencies = [ + "lazy_static", + "regex 1.8.3", + "serde", + "strsim", +] + +[[package]] +name = "filetime" +version = "0.2.21" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5cbc844cecaee9d4443931972e1289c8ff485cb4cc2767cb03ca139ed6885153" +dependencies = [ + "cfg-if 1.0.0", + "libc", + "redox_syscall", + "windows-sys", +] + +[[package]] +name = "getrandom" +version = "0.2.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c85e1d9ab2eadba7e5040d4e09cbd6d072b76a557ad64e797c2cb9d4da21d7e4" +dependencies = [ + "cfg-if 1.0.0", + "libc", + "wasi", +] + +[[package]] +name = "lazy_static" +version = "1.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e2abad23fbc42b3700f2f279844dc832adb2b2eb069b2df918f455c4e18cc646" + +[[package]] +name = "libc" +version = "0.2.144" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2b00cc1c228a6782d0f076e7b232802e0c5689d41bb5df366f2a6b6621cfdfe1" + +[[package]] +name = "libpcre-sys" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0ff3dd28ba96d6fe6752882f2f1b25ba8e1646448e79042442347cf3a92a6666" +dependencies = [ + "bzip2", + "libc", + "pkg-config", + "tar", +] + +[[package]] +name = "memchr" +version = "2.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2dffe52ecf27772e601905b7522cb4ef790d2cc203488bbd0e2fe85fcb74566d" + +[[package]] +name = "memmap" +version = "0.6.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e2ffa2c986de11a9df78620c01eeaaf27d94d3ff02bf81bfcca953102dd0c6ff" +dependencies = [ + "libc", + "winapi", +] + +[[package]] +name = "onig" +version = "3.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f5eeb268a4620c74ea5768c6d2ccd492d60a47a8754666b91a46bfc35cd4d1ba" +dependencies = [ + "bitflags", + "lazy_static", + "libc", + "onig_sys", +] + +[[package]] +name = "onig_sys" +version = "68.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "195ebddbb56740be48042ca117b8fb6e0d99fe392191a9362d82f5f69e510379" +dependencies = [ + "cc", + "libc", + "pkg-config", +] + +[[package]] +name = "pkg-config" +version = "0.3.27" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "26072860ba924cbfa98ea39c8c19b4dd6a4a25423dbdf219c1eca91aa0cf6964" + +[[package]] +name = "proc-macro2" +version = "1.0.59" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6aeca18b86b413c660b781aa319e4e2648a3e6f9eadc9b47e9038e6fe9f3451b" +dependencies = [ + "unicode-ident", +] + +[[package]] +name = "quickcheck" +version = "1.0.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "588f6378e4dd99458b60ec275b4477add41ce4fa9f64dcba6f15adccb19b50d6" +dependencies = [ + "rand", +] + +[[package]] +name = "quote" +version = "1.0.28" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1b9ab9c7eadfd8df19006f1cf1a4aed13540ed5cbc047010ece5826e10825488" +dependencies = [ + "proc-macro2", +] + +[[package]] +name = "rand" +version = "0.8.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "34af8d1a0e25924bc5b7c43c079c942339d8f0a8b57c39049bef581b46327404" +dependencies = [ + "rand_core", +] + +[[package]] +name = "rand_core" +version = "0.6.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ec0be4795e2f6a28069bec0b5ff3e2ac9bafc99e6a9a7dc3547996c5c816922c" +dependencies = [ + "getrandom", +] + +[[package]] +name = "redox_syscall" +version = "0.2.16" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fb5a58c1855b4b6819d59012155603f0b22ad30cad752600aadfcb695265519a" +dependencies = [ + "bitflags", +] + +[[package]] +name = "regex" +version = "1.7.2" +dependencies = [ + "aho-corasick", + "lazy_static", + "memchr", + "quickcheck", + "rand", + "regex-syntax 0.6.29", +] + +[[package]] +name = "regex" +version = "1.8.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "81ca098a9821bd52d6b24fd8b10bd081f47d39c22778cafaa75a2857a62c6390" +dependencies = [ + "regex-syntax 0.7.2", +] + +[[package]] +name = "regex-benchmark" +version = "0.1.0" +dependencies = [ + "cc", + "cfg-if 0.1.10", + "docopt", + "lazy_static", + "libc", + "libpcre-sys", + "memmap", + "onig", + "pkg-config", + "regex 1.7.2", + "regex-syntax 0.6.29", + "serde", +] + +[[package]] +name = "regex-debug" +version = "0.1.0" +dependencies = [ + "docopt", + "regex 1.7.2", + "regex-syntax 0.6.29", + "serde", +] + +[[package]] +name = "regex-syntax" +version = "0.6.29" + +[[package]] +name = "regex-syntax" +version = "0.7.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "436b050e76ed2903236f032a59761c1eb99e1b0aead2c257922771dab1fc8c78" + +[[package]] +name = "rure" +version = "0.2.2" +dependencies = [ + "libc", + "regex 1.7.2", +] + +[[package]] +name = "serde" +version = "1.0.163" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2113ab51b87a539ae008b5c6c02dc020ffa39afd2d83cffcb3f4eb2722cebec2" +dependencies = [ + "serde_derive", +] + +[[package]] +name = "serde_derive" +version = "1.0.163" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8c805777e3930c8883389c602315a24224bcc738b63905ef87cd1420353ea93e" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "strsim" +version = "0.10.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "73473c0e59e6d5812c5dfe2a064a6444949f089e20eec9a2e5506596494e4623" + +[[package]] +name = "syn" +version = "2.0.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "32d41677bcbe24c20c52e7c70b0d8db04134c5d1066bf98662e2871ad200ea3e" +dependencies = [ + "proc-macro2", + "quote", + "unicode-ident", +] + +[[package]] +name = "tar" +version = "0.4.38" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4b55807c0344e1e6c04d7c965f5289c39a8d94ae23ed5c0b57aabac549f871c6" +dependencies = [ + "filetime", + "libc", + "xattr", +] + +[[package]] +name = "unicode-ident" +version = "1.0.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b15811caf2415fb889178633e7724bad2509101cde276048e013b9def5e51fa0" + +[[package]] +name = "wasi" +version = "0.11.0+wasi-snapshot-preview1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9c8d87e72b64a3b4db28d11ce29237c246188f4f51057d65a7eab63b7987e423" + +[[package]] +name = "winapi" +version = "0.3.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5c839a674fcd7a98952e593242ea400abe93992746761e38641405d28b00f419" +dependencies = [ + "winapi-i686-pc-windows-gnu", + "winapi-x86_64-pc-windows-gnu", +] + +[[package]] +name = "winapi-i686-pc-windows-gnu" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ac3b87c63620426dd9b991e5ce0329eff545bccbbb34f3be09ff6fb6ab51b7b6" + +[[package]] +name = "winapi-x86_64-pc-windows-gnu" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "712e227841d057c1ee1cd2fb22fa7e5a5461ae8e48fa2ca79ec42cfc1931183f" + +[[package]] +name = "windows-sys" +version = "0.48.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "677d2418bec65e3338edb076e806bc1ec15693c5d0104683f2efe857f61056a9" +dependencies = [ + "windows-targets", +] + +[[package]] +name = "windows-targets" +version = "0.48.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7b1eb6f0cd7c80c79759c929114ef071b87354ce476d9d94271031c0497adfd5" +dependencies = [ + "windows_aarch64_gnullvm", + "windows_aarch64_msvc", + "windows_i686_gnu", + "windows_i686_msvc", + "windows_x86_64_gnu", + "windows_x86_64_gnullvm", + "windows_x86_64_msvc", +] + +[[package]] +name = "windows_aarch64_gnullvm" +version = "0.48.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "91ae572e1b79dba883e0d315474df7305d12f569b400fcf90581b06062f7e1bc" + +[[package]] +name = "windows_aarch64_msvc" +version = "0.48.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b2ef27e0d7bdfcfc7b868b317c1d32c641a6fe4629c171b8928c7b08d98d7cf3" + +[[package]] +name = "windows_i686_gnu" +version = "0.48.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "622a1962a7db830d6fd0a69683c80a18fda201879f0f447f065a3b7467daa241" + +[[package]] +name = "windows_i686_msvc" +version = "0.48.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4542c6e364ce21bf45d69fdd2a8e455fa38d316158cfd43b3ac1c5b1b19f8e00" + +[[package]] +name = "windows_x86_64_gnu" +version = "0.48.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ca2b8a661f7628cbd23440e50b05d705db3686f894fc9580820623656af974b1" + +[[package]] +name = "windows_x86_64_gnullvm" +version = "0.48.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7896dbc1f41e08872e9d5e8f8baa8fdd2677f29468c4e156210174edc7f7b953" + +[[package]] +name = "windows_x86_64_msvc" +version = "0.48.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1a515f5799fe4961cb532f983ce2b23082366b898e52ffbce459c86f67c8378a" + +[[package]] +name = "xattr" +version = "0.2.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6d1526bbe5aaeb5eb06885f4d987bcdfa5e23187055de9b83fe00156a821fabc" +dependencies = [ + "libc", +] diff --git a/build_sysroot/Cargo.lock b/patches/stdlib-lock.toml similarity index 100% rename from build_sysroot/Cargo.lock rename to patches/stdlib-lock.toml diff --git a/scripts/rustup.sh b/scripts/rustup.sh index 6b446aafb..e62788f2e 100755 --- a/scripts/rustup.sh +++ b/scripts/rustup.sh @@ -33,11 +33,9 @@ case $1 in ./clean_all.sh ./y.sh prepare - - (cd download/sysroot && cargo update && cargo fetch && cp Cargo.lock ../../build_sysroot/) ;; "commit") - git add rust-toolchain build_sysroot/Cargo.lock + git add rust-toolchain git commit -m "Rustup to $(rustc -V)" ;; "push") From eb3e8bb7d729a2df19b1fe2a6a6d7b5e43673807 Mon Sep 17 00:00:00 2001 From: bjorn3 <17426603+bjorn3@users.noreply.github.com> Date: Mon, 29 May 2023 12:16:33 +0000 Subject: [PATCH 13/20] Move some profile settings out of the stdlib cargo workspace patch --- build_system/build_sysroot.rs | 10 +++++++++- build_system/prepare.rs | 8 -------- 2 files changed, 9 insertions(+), 9 deletions(-) diff --git a/build_system/build_sysroot.rs b/build_system/build_sysroot.rs index 1045023c3..5ea34f758 100644 --- a/build_system/build_sysroot.rs +++ b/build_system/build_sysroot.rs @@ -4,7 +4,9 @@ use std::process::{self, Command}; use super::path::{Dirs, RelPath}; use super::rustc_info::{get_file_name, get_rustc_version}; -use super::utils::{remove_dir_if_exists, spawn_and_wait, try_hard_link, CargoProject, Compiler}; +use super::utils::{ + is_ci, remove_dir_if_exists, spawn_and_wait, try_hard_link, CargoProject, Compiler, +}; use super::{CodegenBackend, SysrootKind}; static DIST_DIR: RelPath = RelPath::DIST; @@ -272,10 +274,16 @@ fn build_clif_sysroot_for_triple( } compiler.rustflags += &rustflags; let mut build_cmd = STANDARD_LIBRARY.build(&compiler, dirs); + build_cmd.env("CARGO_BUILD_INCREMENTAL", "true"); // Force incr comp even in release mode + if is_ci() { + // Disabling incr comp reduces cache size and incr comp doesn't save as much on CI anyway + build_cmd.env("CARGO_BUILD_INCREMENTAL", "false"); + } if channel == "release" { build_cmd.arg("--release"); } build_cmd.arg("--features").arg("compiler-builtins-no-asm backtrace panic-unwind"); + build_cmd.env("CARGO_PROFILE_RELEASE_DEBUG", "true"); build_cmd.env("__CARGO_DEFAULT_LIB_METADATA", "cg_clif"); if compiler.triple.contains("apple") { build_cmd.env("CARGO_PROFILE_RELEASE_SPLIT_DEBUGINFO", "packed"); diff --git a/build_system/prepare.rs b/build_system/prepare.rs index c6bc796f3..350c3ad8f 100644 --- a/build_system/prepare.rs +++ b/build_system/prepare.rs @@ -37,14 +37,6 @@ rustc-std-workspace-core = { path = "./library/rustc-std-workspace-core" } rustc-std-workspace-alloc = { path = "./library/rustc-std-workspace-alloc" } rustc-std-workspace-std = { path = "./library/rustc-std-workspace-std" } -[profile.dev] -lto = "off" - -[profile.release] -debug = true -incremental = true -lto = "off" - # Mandatory for correctly compiling compiler-builtins [profile.dev.package.compiler_builtins] debug-assertions = false From 6b9af8cb36d2bc1ae57768e3c8703093870d919f Mon Sep 17 00:00:00 2001 From: bjorn3 <17426603+bjorn3@users.noreply.github.com> Date: Mon, 29 May 2023 13:18:41 +0000 Subject: [PATCH 14/20] Disable all incremental compilation for CARGO_BUILD_INCREMENTAL=false --- build_system/build_backend.rs | 8 ++------ build_system/build_sysroot.rs | 8 ++------ build_system/utils.rs | 10 ++++++++++ 3 files changed, 14 insertions(+), 12 deletions(-) diff --git a/build_system/build_backend.rs b/build_system/build_backend.rs index b88489a34..6855c1a7f 100644 --- a/build_system/build_backend.rs +++ b/build_system/build_backend.rs @@ -3,7 +3,7 @@ use std::path::PathBuf; use super::path::{Dirs, RelPath}; use super::rustc_info::get_file_name; -use super::utils::{is_ci, is_ci_opt, CargoProject, Compiler}; +use super::utils::{is_ci, is_ci_opt, maybe_incremental, CargoProject, Compiler}; pub(crate) static CG_CLIF: CargoProject = CargoProject::new(&RelPath::SOURCE, "cg_clif"); @@ -14,8 +14,7 @@ pub(crate) fn build_backend( use_unstable_features: bool, ) -> PathBuf { let mut cmd = CG_CLIF.build(&bootstrap_host_compiler, dirs); - - cmd.env("CARGO_BUILD_INCREMENTAL", "true"); // Force incr comp even in release mode + maybe_incremental(&mut cmd); let mut rustflags = env::var("RUSTFLAGS").unwrap_or_default(); @@ -23,9 +22,6 @@ pub(crate) fn build_backend( // Deny warnings on CI rustflags += " -Dwarnings"; - // Disabling incr comp reduces cache size and incr comp doesn't save as much on CI anyway - cmd.env("CARGO_BUILD_INCREMENTAL", "false"); - if !is_ci_opt() { cmd.env("CARGO_PROFILE_RELEASE_DEBUG_ASSERTIONS", "true"); cmd.env("CARGO_PROFILE_RELEASE_OVERFLOW_CHECKS", "true"); diff --git a/build_system/build_sysroot.rs b/build_system/build_sysroot.rs index 5ea34f758..04cd3894a 100644 --- a/build_system/build_sysroot.rs +++ b/build_system/build_sysroot.rs @@ -5,7 +5,7 @@ use std::process::{self, Command}; use super::path::{Dirs, RelPath}; use super::rustc_info::{get_file_name, get_rustc_version}; use super::utils::{ - is_ci, remove_dir_if_exists, spawn_and_wait, try_hard_link, CargoProject, Compiler, + maybe_incremental, remove_dir_if_exists, spawn_and_wait, try_hard_link, CargoProject, Compiler, }; use super::{CodegenBackend, SysrootKind}; @@ -274,11 +274,7 @@ fn build_clif_sysroot_for_triple( } compiler.rustflags += &rustflags; let mut build_cmd = STANDARD_LIBRARY.build(&compiler, dirs); - build_cmd.env("CARGO_BUILD_INCREMENTAL", "true"); // Force incr comp even in release mode - if is_ci() { - // Disabling incr comp reduces cache size and incr comp doesn't save as much on CI anyway - build_cmd.env("CARGO_BUILD_INCREMENTAL", "false"); - } + maybe_incremental(&mut build_cmd); if channel == "release" { build_cmd.arg("--release"); } diff --git a/build_system/utils.rs b/build_system/utils.rs index e7fcb2644..41fc366e2 100644 --- a/build_system/utils.rs +++ b/build_system/utils.rs @@ -258,3 +258,13 @@ pub(crate) fn is_ci() -> bool { pub(crate) fn is_ci_opt() -> bool { env::var("CI_OPT").is_ok() } + +pub(crate) fn maybe_incremental(cmd: &mut Command) { + if is_ci() || std::env::var("CARGO_BUILD_INCREMENTAL").map_or(false, |val| val == "false") { + // Disabling incr comp reduces cache size and incr comp doesn't save as much on CI anyway + cmd.env("CARGO_BUILD_INCREMENTAL", "false"); + } else { + // Force incr comp even in release mode unless in CI or incremental builds are explicitly disabled + cmd.env("CARGO_BUILD_INCREMENTAL", "true"); + } +} From d0ea8bbc5e8dcb2018a248ee9d88a02ce2319b3c Mon Sep 17 00:00:00 2001 From: bjorn3 <17426603+bjorn3@users.noreply.github.com> Date: Mon, 29 May 2023 13:41:55 +0000 Subject: [PATCH 15/20] Only copy library dir for stdlib When building as part of rust, the sysroot source dir is symlinked to the main source dir, which contains the build dir to which we are likely copying. --- build_system/prepare.rs | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/build_system/prepare.rs b/build_system/prepare.rs index 350c3ad8f..748d92614 100644 --- a/build_system/prepare.rs +++ b/build_system/prepare.rs @@ -268,7 +268,12 @@ pub(crate) fn apply_patches(dirs: &Dirs, crate_name: &str, source_dir: &Path, ta remove_dir_if_exists(target_dir); fs::create_dir_all(target_dir).unwrap(); - copy_dir_recursively(source_dir, target_dir); + if crate_name == "stdlib" { + fs::create_dir(target_dir.join("library")).unwrap(); + copy_dir_recursively(&source_dir.join("library"), &target_dir.join("library")); + } else { + copy_dir_recursively(source_dir, target_dir); + } init_git_repo(target_dir); From bcac2220134a0b6ca858b674c00ad809c2ed446a Mon Sep 17 00:00:00 2001 From: bjorn3 <17426603+bjorn3@users.noreply.github.com> Date: Sun, 4 Jun 2023 15:41:43 +0000 Subject: [PATCH 16/20] Lazily patch the standard library --- build_system/build_sysroot.rs | 27 ++++++--------------------- build_system/main.rs | 23 ++++++++++++++++------- build_system/prepare.rs | 14 ++++---------- build_system/rustc_info.rs | 6 ------ 4 files changed, 26 insertions(+), 44 deletions(-) diff --git a/build_system/build_sysroot.rs b/build_system/build_sysroot.rs index 04cd3894a..74bba9ed5 100644 --- a/build_system/build_sysroot.rs +++ b/build_system/build_sysroot.rs @@ -1,9 +1,9 @@ use std::fs; use std::path::{Path, PathBuf}; -use std::process::{self, Command}; +use std::process::Command; use super::path::{Dirs, RelPath}; -use super::rustc_info::{get_file_name, get_rustc_version}; +use super::rustc_info::get_file_name; use super::utils::{ maybe_incremental, remove_dir_if_exists, spawn_and_wait, try_hard_link, CargoProject, Compiler, }; @@ -158,7 +158,6 @@ impl SysrootTarget { } pub(crate) static STDLIB_SRC: RelPath = RelPath::BUILD.join("stdlib"); -pub(crate) static SYSROOT_RUSTC_VERSION: RelPath = STDLIB_SRC.join("rustc_version"); pub(crate) static STANDARD_LIBRARY: CargoProject = CargoProject::new(&STDLIB_SRC.join("library/sysroot"), "stdlib_target"); pub(crate) static RTSTARTUP_SYSROOT: RelPath = RelPath::BUILD.join("rtstartup"); @@ -222,24 +221,6 @@ fn build_clif_sysroot_for_triple( mut compiler: Compiler, cg_clif_dylib_path: &CodegenBackend, ) -> SysrootTarget { - match fs::read_to_string(SYSROOT_RUSTC_VERSION.to_path(dirs)) { - Err(e) => { - eprintln!("Failed to get rustc version for patched sysroot source: {}", e); - eprintln!("Hint: Try `./y.sh prepare` to patch the sysroot source"); - process::exit(1); - } - Ok(source_version) => { - let rustc_version = get_rustc_version(&compiler.rustc); - if source_version != rustc_version { - eprintln!("The patched sysroot source is outdated"); - eprintln!("Source version: {}", source_version.trim()); - eprintln!("Rustc version: {}", rustc_version.trim()); - eprintln!("Hint: Try `./y.sh prepare` to update the patched sysroot source"); - process::exit(1); - } - } - } - let mut target_libs = SysrootTarget { triple: compiler.triple.clone(), libs: vec![] }; if let Some(rtstartup_target_libs) = build_rtstartup(dirs, &compiler) { @@ -302,6 +283,10 @@ fn build_clif_sysroot_for_triple( } fn build_rtstartup(dirs: &Dirs, compiler: &Compiler) -> Option { + if !super::config::get_bool("keep_sysroot") { + super::prepare::prepare_stdlib(dirs, &compiler.rustc); + } + if !compiler.triple.ends_with("windows-gnu") { return None; } diff --git a/build_system/main.rs b/build_system/main.rs index 06395eb14..d51e5027c 100644 --- a/build_system/main.rs +++ b/build_system/main.rs @@ -126,6 +126,22 @@ fn main() { } } + let current_dir = std::env::current_dir().unwrap(); + out_dir = current_dir.join(out_dir); + + if command == Command::Prepare { + prepare::prepare(&path::Dirs { + source_dir: current_dir.clone(), + download_dir: download_dir + .map(|dir| current_dir.join(dir)) + .unwrap_or_else(|| out_dir.join("download")), + build_dir: PathBuf::from("dummy_do_not_use"), + dist_dir: PathBuf::from("dummy_do_not_use"), + frozen, + }); + process::exit(0); + } + let rustup_toolchain_name = match (env::var("CARGO"), env::var("RUSTC"), env::var("RUSTDOC")) { (Ok(_), Ok(_), Ok(_)) => None, (Err(_), Err(_), Err(_)) => Some(rustc_info::get_toolchain_name()), @@ -158,8 +174,6 @@ fn main() { .unwrap_or_else(|| bootstrap_host_compiler.triple.clone()); // FIXME allow changing the location of these dirs using cli arguments - let current_dir = std::env::current_dir().unwrap(); - out_dir = current_dir.join(out_dir); let dirs = path::Dirs { source_dir: current_dir.clone(), download_dir: download_dir @@ -181,11 +195,6 @@ fn main() { std::fs::File::create(target).unwrap(); } - if command == Command::Prepare { - prepare::prepare(&dirs, &bootstrap_host_compiler.rustc); - process::exit(0); - } - env::set_var("RUSTC", "rustc_should_be_set_explicitly"); env::set_var("RUSTDOC", "rustdoc_should_be_set_explicitly"); diff --git a/build_system/prepare.rs b/build_system/prepare.rs index 748d92614..77f7175b7 100644 --- a/build_system/prepare.rs +++ b/build_system/prepare.rs @@ -3,24 +3,21 @@ use std::fs; use std::path::{Path, PathBuf}; use std::process::Command; -use super::build_sysroot::{STDLIB_SRC, SYSROOT_RUSTC_VERSION}; +use super::build_sysroot::STDLIB_SRC; use super::path::{Dirs, RelPath}; -use super::rustc_info::{get_default_sysroot, get_rustc_version}; +use super::rustc_info::get_default_sysroot; use super::utils::{ copy_dir_recursively, git_command, remove_dir_if_exists, retry_spawn_and_wait, spawn_and_wait, }; -pub(crate) fn prepare(dirs: &Dirs, rustc: &Path) { +pub(crate) fn prepare(dirs: &Dirs) { RelPath::DOWNLOAD.ensure_exists(dirs); super::tests::RAND_REPO.fetch(dirs); super::tests::REGEX_REPO.fetch(dirs); super::tests::PORTABLE_SIMD_REPO.fetch(dirs); - - // FIXME do this on the fly? - prepare_stdlib(dirs, rustc); } -fn prepare_stdlib(dirs: &Dirs, rustc: &Path) { +pub(crate) fn prepare_stdlib(dirs: &Dirs, rustc: &Path) { let sysroot_src_orig = get_default_sysroot(rustc).join("lib/rustlib/src/rust"); assert!(sysroot_src_orig.exists()); @@ -50,9 +47,6 @@ codegen-units = 10000 "#, ) .unwrap(); - - let rustc_version = get_rustc_version(rustc); - fs::write(SYSROOT_RUSTC_VERSION.to_path(dirs), &rustc_version).unwrap(); } pub(crate) struct GitRepo { diff --git a/build_system/rustc_info.rs b/build_system/rustc_info.rs index 42cec0c69..5b71504e9 100644 --- a/build_system/rustc_info.rs +++ b/build_system/rustc_info.rs @@ -1,12 +1,6 @@ use std::path::{Path, PathBuf}; use std::process::{Command, Stdio}; -pub(crate) fn get_rustc_version(rustc: &Path) -> String { - let version_info = - Command::new(rustc).stderr(Stdio::inherit()).args(&["-V"]).output().unwrap().stdout; - String::from_utf8(version_info).unwrap() -} - pub(crate) fn get_host_triple(rustc: &Path) -> String { let version_info = Command::new(rustc).stderr(Stdio::inherit()).args(&["-vV"]).output().unwrap().stdout; From 8c1c84d79e26e5250d3ad1ee894843c01785787a Mon Sep 17 00:00:00 2001 From: bjorn3 <17426603+bjorn3@users.noreply.github.com> Date: Mon, 5 Jun 2023 13:59:08 +0000 Subject: [PATCH 17/20] Copy Cargo.lock over in ./y.sh prepare This makes it easier for ./x.py to vendor all dependencies --- build_system/prepare.rs | 21 +++++++++++++-------- build_system/tests.rs | 10 +++++++--- 2 files changed, 20 insertions(+), 11 deletions(-) diff --git a/build_system/prepare.rs b/build_system/prepare.rs index 77f7175b7..e31e39a48 100644 --- a/build_system/prepare.rs +++ b/build_system/prepare.rs @@ -47,6 +47,10 @@ codegen-units = 10000 "#, ) .unwrap(); + + let source_lockfile = RelPath::PATCHES.to_path(dirs).join("stdlib-lock.toml"); + let target_lockfile = STDLIB_SRC.to_path(dirs).join("Cargo.lock"); + fs::copy(source_lockfile, target_lockfile).unwrap(); } pub(crate) struct GitRepo { @@ -134,6 +138,15 @@ impl GitRepo { } } + let source_lockfile = + RelPath::PATCHES.to_path(dirs).join(format!("{}-lock.toml", self.patch_name)); + let target_lockfile = download_dir.join("Cargo.lock"); + if source_lockfile.exists() { + fs::copy(source_lockfile, target_lockfile).unwrap(); + } else { + assert!(target_lockfile.exists()); + } + let actual_hash = format!("{:016x}", hash_dir(&download_dir)); if actual_hash != self.content_hash { println!( @@ -285,12 +298,4 @@ pub(crate) fn apply_patches(dirs: &Dirs, crate_name: &str, source_dir: &Path, ta apply_patch_cmd.arg(patch).arg("-q"); spawn_and_wait(apply_patch_cmd); } - - let source_lockfile = RelPath::PATCHES.to_path(dirs).join(format!("{crate_name}-lock.toml")); - let target_lockfile = target_dir.join("Cargo.lock"); - if source_lockfile.exists() { - fs::copy(source_lockfile, target_lockfile).unwrap(); - } else { - assert!(target_lockfile.exists()); - } } diff --git a/build_system/tests.rs b/build_system/tests.rs index d104efd61..733a57240 100644 --- a/build_system/tests.rs +++ b/build_system/tests.rs @@ -100,7 +100,7 @@ pub(crate) static RAND_REPO: GitRepo = GitRepo::github( "rust-random", "rand", "50b9a447410860af8d6db9a208c3576886955874", - "98b2276210b30e43", + "446203b96054891e", "rand", ); @@ -110,7 +110,7 @@ pub(crate) static REGEX_REPO: GitRepo = GitRepo::github( "rust-lang", "regex", "32fed9429eafba0ae92a64b01796a0c5a75b88c8", - "d6af6507d565aa66", + "fcc4df7c5b902633", "regex", ); @@ -120,7 +120,7 @@ pub(crate) static PORTABLE_SIMD_REPO: GitRepo = GitRepo::github( "rust-lang", "portable-simd", "ad8afa8c81273b3b49acbea38cd3bcf17a34cf2b", - "1ba291009510070b", + "800548f8000e31bd", "portable-simd", ); @@ -156,6 +156,10 @@ const EXTENDED_SYSROOT_SUITE: &[TestCase] = &[ &LIBCORE_TESTS_SRC.to_path(&runner.dirs), ); + let source_lockfile = RelPath::PATCHES.to_path(&runner.dirs).join("coretests-lock.toml"); + let target_lockfile = LIBCORE_TESTS_SRC.to_path(&runner.dirs).join("Cargo.lock"); + fs::copy(source_lockfile, target_lockfile).unwrap(); + LIBCORE_TESTS.clean(&runner.dirs); if runner.is_native { From a691b14aeeea6de133c23050e33732b5fe620a7b Mon Sep 17 00:00:00 2001 From: bjorn3 <17426603+bjorn3@users.noreply.github.com> Date: Mon, 5 Jun 2023 15:15:48 +0000 Subject: [PATCH 18/20] Allow skipping tests from the commandline --- build_system/main.rs | 8 ++++++++ build_system/tests.rs | 27 +++++++++++++++++++-------- build_system/usage.txt | 5 ++++- 3 files changed, 31 insertions(+), 9 deletions(-) diff --git a/build_system/main.rs b/build_system/main.rs index d51e5027c..41d0d6f23 100644 --- a/build_system/main.rs +++ b/build_system/main.rs @@ -90,6 +90,7 @@ fn main() { let mut sysroot_kind = SysrootKind::Clif; let mut use_unstable_features = true; let mut frozen = false; + let mut skip_tests = vec![]; let mut use_backend = None; while let Some(arg) = args.next().as_deref() { match arg { @@ -115,6 +116,12 @@ fn main() { } "--no-unstable-features" => use_unstable_features = false, "--frozen" => frozen = true, + "--skip-test" => { + // FIXME check that all passed in tests actually exist + skip_tests.push(args.next().unwrap_or_else(|| { + arg_error!("--skip-test requires argument"); + })); + } "--use-backend" => { use_backend = Some(match args.next() { Some(name) => name, @@ -218,6 +225,7 @@ fn main() { channel, sysroot_kind, use_unstable_features, + &skip_tests.iter().map(|test| &**test).collect::>(), &cg_clif_dylib, &bootstrap_host_compiler, rustup_toolchain_name.as_deref(), diff --git a/build_system/tests.rs b/build_system/tests.rs index 733a57240..08d8f708c 100644 --- a/build_system/tests.rs +++ b/build_system/tests.rs @@ -19,7 +19,7 @@ struct TestCase { } enum TestCaseCmd { - Custom { func: &'static dyn Fn(&TestRunner) }, + Custom { func: &'static dyn Fn(&TestRunner<'_>) }, BuildLib { source: &'static str, crate_types: &'static str }, BuildBinAndRun { source: &'static str, args: &'static [&'static str] }, JitBin { source: &'static str, args: &'static str }, @@ -27,7 +27,7 @@ enum TestCaseCmd { impl TestCase { // FIXME reduce usage of custom test case commands - const fn custom(config: &'static str, func: &'static dyn Fn(&TestRunner)) -> Self { + const fn custom(config: &'static str, func: &'static dyn Fn(&TestRunner<'_>)) -> Self { Self { config, cmd: TestCaseCmd::Custom { func } } } @@ -247,6 +247,7 @@ pub(crate) fn run_tests( channel: &str, sysroot_kind: SysrootKind, use_unstable_features: bool, + skip_tests: &[&str], cg_clif_dylib: &CodegenBackend, bootstrap_host_compiler: &Compiler, rustup_toolchain_name: Option<&str>, @@ -256,7 +257,7 @@ pub(crate) fn run_tests( get_default_sysroot(&bootstrap_host_compiler.rustc).join("lib/rustlib/src/rust"); assert!(stdlib_source.exists()); - if config::get_bool("testsuite.no_sysroot") { + if config::get_bool("testsuite.no_sysroot") && !skip_tests.contains(&"testsuite.no_sysroot") { let target_compiler = build_sysroot::build_sysroot( dirs, channel, @@ -271,6 +272,7 @@ pub(crate) fn run_tests( dirs.clone(), target_compiler, use_unstable_features, + skip_tests, bootstrap_host_compiler.triple == target_triple, stdlib_source.clone(), ); @@ -281,8 +283,10 @@ pub(crate) fn run_tests( eprintln!("[SKIP] no_sysroot tests"); } - let run_base_sysroot = config::get_bool("testsuite.base_sysroot"); - let run_extended_sysroot = config::get_bool("testsuite.extended_sysroot"); + let run_base_sysroot = config::get_bool("testsuite.base_sysroot") + && !skip_tests.contains(&"testsuite.base_sysroot"); + let run_extended_sysroot = config::get_bool("testsuite.extended_sysroot") + && !skip_tests.contains(&"testsuite.extended_sysroot"); if run_base_sysroot || run_extended_sysroot { let mut target_compiler = build_sysroot::build_sysroot( @@ -302,6 +306,7 @@ pub(crate) fn run_tests( dirs.clone(), target_compiler, use_unstable_features, + skip_tests, bootstrap_host_compiler.triple == target_triple, stdlib_source, ); @@ -320,20 +325,22 @@ pub(crate) fn run_tests( } } -struct TestRunner { +struct TestRunner<'a> { is_native: bool, jit_supported: bool, use_unstable_features: bool, + skip_tests: &'a [&'a str], dirs: Dirs, target_compiler: Compiler, stdlib_source: PathBuf, } -impl TestRunner { +impl<'a> TestRunner<'a> { fn new( dirs: Dirs, mut target_compiler: Compiler, use_unstable_features: bool, + skip_tests: &'a [&'a str], is_native: bool, stdlib_source: PathBuf, ) -> Self { @@ -360,6 +367,7 @@ impl TestRunner { is_native, jit_supported, use_unstable_features, + skip_tests, dirs, target_compiler, stdlib_source, @@ -372,7 +380,10 @@ impl TestRunner { let tag = tag.to_uppercase(); let is_jit_test = tag == "JIT"; - if !config::get_bool(config) || (is_jit_test && !self.jit_supported) { + if !config::get_bool(config) + || (is_jit_test && !self.jit_supported) + || self.skip_tests.contains(&config) + { eprintln!("[{tag}] {testname} (skipped)"); continue; } else { diff --git a/build_system/usage.txt b/build_system/usage.txt index 9d20cdca6..6d3b3a13d 100644 --- a/build_system/usage.txt +++ b/build_system/usage.txt @@ -3,7 +3,7 @@ The build system of cg_clif. USAGE: ./y.sh prepare [--out-dir DIR] [--download-dir DIR] ./y.sh build [--debug] [--sysroot none|clif|llvm] [--out-dir DIR] [--download-dir DIR] [--no-unstable-features] [--frozen] - ./y.sh test [--debug] [--sysroot none|clif|llvm] [--out-dir DIR] [--download-dir DIR] [--no-unstable-features] [--frozen] + ./y.sh test [--debug] [--sysroot none|clif|llvm] [--out-dir DIR] [--download-dir DIR] [--no-unstable-features] [--frozen] [--skip-test TESTNAME] ./y.sh abi-cafe [--debug] [--sysroot none|clif|llvm] [--out-dir DIR] [--download-dir DIR] [--no-unstable-features] [--frozen] ./y.sh bench [--debug] [--sysroot none|clif|llvm] [--out-dir DIR] [--download-dir DIR] [--no-unstable-features] [--frozen] @@ -32,6 +32,9 @@ OPTIONS: --frozen Require Cargo.lock and cache are up to date + --skip-test TESTNAME + Skip testing the TESTNAME test. The test name format is the same as config.txt. + --use-backend NAME Use the existing Cranelift (or other) backend of the rustc with which we built. Warning: This is meant for use in rust's CI only! From 485d7e129f5beb67e7769014959e07aff5989d22 Mon Sep 17 00:00:00 2001 From: bjorn3 <17426603+bjorn3@users.noreply.github.com> Date: Mon, 5 Jun 2023 19:06:15 +0000 Subject: [PATCH 19/20] Remove outdated fixme --- build_system/main.rs | 1 - 1 file changed, 1 deletion(-) diff --git a/build_system/main.rs b/build_system/main.rs index 41d0d6f23..3bc78d5db 100644 --- a/build_system/main.rs +++ b/build_system/main.rs @@ -180,7 +180,6 @@ fn main() { .or_else(|| config::get_value("target")) .unwrap_or_else(|| bootstrap_host_compiler.triple.clone()); - // FIXME allow changing the location of these dirs using cli arguments let dirs = path::Dirs { source_dir: current_dir.clone(), download_dir: download_dir From e9bd63af3cfc192f0f998debbf5ab3e597781cec Mon Sep 17 00:00:00 2001 From: bjorn3 <17426603+bjorn3@users.noreply.github.com> Date: Thu, 15 Jun 2023 17:14:49 +0000 Subject: [PATCH 20/20] Ignore -Clink-arg=-import-instr-limit --- src/config.rs | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/src/config.rs b/src/config.rs index 263401e1c..9e92d656c 100644 --- a/src/config.rs +++ b/src/config.rs @@ -82,6 +82,11 @@ impl BackendConfig { let mut config = BackendConfig::default(); for opt in opts { + if opt.starts_with("-import-instr-limit") { + // Silently ignore -import-instr-limit. It is set by rust's build system even when + // testing cg_clif. + continue; + } if let Some((name, value)) = opt.split_once('=') { match name { "mode" => config.codegen_mode = value.parse()?,