Skip to content

Commit cbcf9a5

Browse files
borsbors
committed
Auto merge of #115795 - Kobzol:opt-dist-custom, r=Mark-Simulacrum
Refactor `opt-dist` to simplify local building This PR refactors the `opt-dist` tool to make it easier to invoke it locally, outside of CI, and thus simplify building PGO/BOLT optimized `rustc` builds e.g. for distro maintainers. It should also make it easier to run the PGO/BOLT workflow locally e.g. to profile performance or debug issues (looking at you, #115554).
2 parents de68911 + ee451f8 commit cbcf9a5

File tree

15 files changed

+423
-277
lines changed

15 files changed

+423
-277
lines changed

Diff for: ‎.github/workflows/ci.yml

+1-1
Original file line numberDiff line numberDiff line change
@@ -393,7 +393,7 @@ jobs:
393393
- name: dist-x86_64-msvc
394394
env:
395395
RUST_CONFIGURE_ARGS: "--build=x86_64-pc-windows-msvc --host=x86_64-pc-windows-msvc --target=x86_64-pc-windows-msvc --enable-full-tools --enable-profiler"
396-
SCRIPT: python x.py build --set rust.debug=true opt-dist && PGO_HOST=x86_64-pc-windows-msvc ./build/x86_64-pc-windows-msvc/stage0-tools-bin/opt-dist python x.py dist bootstrap --include-default-paths
396+
SCRIPT: python x.py build --set rust.debug=true opt-dist && PGO_HOST=x86_64-pc-windows-msvc ./build/x86_64-pc-windows-msvc/stage0-tools-bin/opt-dist windows-ci -- python x.py dist bootstrap --include-default-paths
397397
DIST_REQUIRE_ALL_TOOLS: 1
398398
os: windows-2019-8core-32gb
399399
- name: dist-i686-msvc

Diff for: ‎Cargo.lock

+72-4
Original file line numberDiff line numberDiff line change
@@ -858,14 +858,38 @@ dependencies = [
858858
"winapi",
859859
]
860860

861+
[[package]]
862+
name = "darling"
863+
version = "0.14.4"
864+
source = "registry+https://github.com/rust-lang/crates.io-index"
865+
checksum = "7b750cb3417fd1b327431a470f388520309479ab0bf5e323505daf0290cd3850"
866+
dependencies = [
867+
"darling_core 0.14.4",
868+
"darling_macro 0.14.4",
869+
]
870+
861871
[[package]]
862872
name = "darling"
863873
version = "0.20.3"
864874
source = "registry+https://github.com/rust-lang/crates.io-index"
865875
checksum = "0209d94da627ab5605dcccf08bb18afa5009cfbef48d8a8b7d7bdbc79be25c5e"
866876
dependencies = [
867-
"darling_core",
868-
"darling_macro",
877+
"darling_core 0.20.3",
878+
"darling_macro 0.20.3",
879+
]
880+
881+
[[package]]
882+
name = "darling_core"
883+
version = "0.14.4"
884+
source = "registry+https://github.com/rust-lang/crates.io-index"
885+
checksum = "109c1ca6e6b7f82cc233a97004ea8ed7ca123a9af07a8230878fcfda9b158bf0"
886+
dependencies = [
887+
"fnv",
888+
"ident_case",
889+
"proc-macro2",
890+
"quote",
891+
"strsim",
892+
"syn 1.0.109",
869893
]
870894

871895
[[package]]
@@ -882,13 +906,24 @@ dependencies = [
882906
"syn 2.0.29",
883907
]
884908

909+
[[package]]
910+
name = "darling_macro"
911+
version = "0.14.4"
912+
source = "registry+https://github.com/rust-lang/crates.io-index"
913+
checksum = "a4aab4dbc9f7611d8b55048a3a16d2d010c2c8334e46304b40ac1cc14bf3b48e"
914+
dependencies = [
915+
"darling_core 0.14.4",
916+
"quote",
917+
"syn 1.0.109",
918+
]
919+
885920
[[package]]
886921
name = "darling_macro"
887922
version = "0.20.3"
888923
source = "registry+https://github.com/rust-lang/crates.io-index"
889924
checksum = "836a9bbc7ad63342d6d6e7b815ccab164bc77a2d95d84bc3117a8c0d5c98e2d5"
890925
dependencies = [
891-
"darling_core",
926+
"darling_core 0.20.3",
892927
"quote",
893928
"syn 2.0.29",
894929
]
@@ -919,6 +954,37 @@ dependencies = [
919954
"syn 1.0.109",
920955
]
921956

957+
[[package]]
958+
name = "derive_builder"
959+
version = "0.12.0"
960+
source = "registry+https://github.com/rust-lang/crates.io-index"
961+
checksum = "8d67778784b508018359cbc8696edb3db78160bab2c2a28ba7f56ef6932997f8"
962+
dependencies = [
963+
"derive_builder_macro",
964+
]
965+
966+
[[package]]
967+
name = "derive_builder_core"
968+
version = "0.12.0"
969+
source = "registry+https://github.com/rust-lang/crates.io-index"
970+
checksum = "c11bdc11a0c47bc7d37d582b5285da6849c96681023680b906673c5707af7b0f"
971+
dependencies = [
972+
"darling 0.14.4",
973+
"proc-macro2",
974+
"quote",
975+
"syn 1.0.109",
976+
]
977+
978+
[[package]]
979+
name = "derive_builder_macro"
980+
version = "0.12.0"
981+
source = "registry+https://github.com/rust-lang/crates.io-index"
982+
checksum = "ebcda35c7a396850a55ffeac740804b40ffec779b98fffbb1738f4033f0ee79e"
983+
dependencies = [
984+
"derive_builder_core",
985+
"syn 1.0.109",
986+
]
987+
922988
[[package]]
923989
name = "derive_more"
924990
version = "0.99.17"
@@ -938,7 +1004,7 @@ version = "0.1.6"
9381004
source = "registry+https://github.com/rust-lang/crates.io-index"
9391005
checksum = "4e8ef033054e131169b8f0f9a7af8f5533a9436fadf3c500ed547f730f07090d"
9401006
dependencies = [
941-
"darling",
1007+
"darling 0.20.3",
9421008
"proc-macro2",
9431009
"quote",
9441010
"syn 2.0.29",
@@ -2584,6 +2650,8 @@ dependencies = [
25842650
"anyhow",
25852651
"build_helper",
25862652
"camino",
2653+
"clap",
2654+
"derive_builder",
25872655
"env_logger 0.10.0",
25882656
"fs_extra",
25892657
"glob",

Diff for: ‎src/ci/docker/host-x86_64/dist-x86_64-linux/Dockerfile

+1-1
Original file line numberDiff line numberDiff line change
@@ -87,7 +87,7 @@ ENV RUST_CONFIGURE_ARGS \
8787
--set rust.lto=thin
8888

8989
ENV SCRIPT python3 ../x.py build --set rust.debug=true opt-dist && \
90-
./build/$HOSTS/stage0-tools-bin/opt-dist python3 ../x.py dist \
90+
./build/$HOSTS/stage0-tools-bin/opt-dist linux-ci -- python3 ../x.py dist \
9191
--host $HOSTS --target $HOSTS \
9292
--include-default-paths \
9393
build-manifest bootstrap

Diff for: ‎src/ci/github-actions/ci.yml

+1-1
Original file line numberDiff line numberDiff line change
@@ -624,7 +624,7 @@ jobs:
624624
--target=x86_64-pc-windows-msvc
625625
--enable-full-tools
626626
--enable-profiler
627-
SCRIPT: python x.py build --set rust.debug=true opt-dist && PGO_HOST=x86_64-pc-windows-msvc ./build/x86_64-pc-windows-msvc/stage0-tools-bin/opt-dist python x.py dist bootstrap --include-default-paths
627+
SCRIPT: python x.py build --set rust.debug=true opt-dist && PGO_HOST=x86_64-pc-windows-msvc ./build/x86_64-pc-windows-msvc/stage0-tools-bin/opt-dist windows-ci -- python x.py dist bootstrap --include-default-paths
628628
DIST_REQUIRE_ALL_TOOLS: 1
629629
<<: *job-windows-8c
630630

Diff for: ‎src/tools/opt-dist/Cargo.toml

+2
Original file line numberDiff line numberDiff line change
@@ -21,3 +21,5 @@ serde = { version = "1", features = ["derive"] }
2121
serde_json = "1"
2222
glob = "0.3"
2323
tempfile = "3.5"
24+
derive_builder = "0.12"
25+
clap = { version = "4", features = ["derive"] }

Diff for: ‎src/tools/opt-dist/src/environment.rs

+108
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,108 @@
1+
use camino::Utf8PathBuf;
2+
use derive_builder::Builder;
3+
4+
#[derive(Builder)]
5+
pub struct Environment {
6+
host_triple: String,
7+
python_binary: String,
8+
/// The rustc checkout, where the compiler source is located.
9+
checkout_dir: Utf8PathBuf,
10+
/// The main directory where the build occurs. Stage0 rustc and cargo have to be available in
11+
/// this directory before `opt-dist` is started.
12+
build_dir: Utf8PathBuf,
13+
/// Directory where the optimization artifacts (PGO/BOLT profiles, etc.)
14+
/// will be stored.
15+
artifact_dir: Utf8PathBuf,
16+
/// Path to the host LLVM used to compile LLVM in `src/llvm-project`.
17+
host_llvm_dir: Utf8PathBuf,
18+
/// List of test paths that should be skipped when testing the optimized artifacts.
19+
skipped_tests: Vec<String>,
20+
/// Directory containing a pre-built rustc-perf checkout.
21+
#[builder(default)]
22+
prebuilt_rustc_perf: Option<Utf8PathBuf>,
23+
use_bolt: bool,
24+
shared_llvm: bool,
25+
}
26+
27+
impl Environment {
28+
pub fn host_triple(&self) -> &str {
29+
&self.host_triple
30+
}
31+
32+
pub fn python_binary(&self) -> &str {
33+
&self.python_binary
34+
}
35+
36+
pub fn checkout_path(&self) -> Utf8PathBuf {
37+
self.checkout_dir.clone()
38+
}
39+
40+
pub fn build_root(&self) -> Utf8PathBuf {
41+
self.build_dir.clone()
42+
}
43+
44+
pub fn build_artifacts(&self) -> Utf8PathBuf {
45+
self.build_root().join("build").join(&self.host_triple)
46+
}
47+
48+
pub fn artifact_dir(&self) -> Utf8PathBuf {
49+
self.artifact_dir.clone()
50+
}
51+
52+
pub fn cargo_stage_0(&self) -> Utf8PathBuf {
53+
self.build_artifacts()
54+
.join("stage0")
55+
.join("bin")
56+
.join(format!("cargo{}", executable_extension()))
57+
}
58+
59+
pub fn rustc_stage_0(&self) -> Utf8PathBuf {
60+
self.build_artifacts()
61+
.join("stage0")
62+
.join("bin")
63+
.join(format!("rustc{}", executable_extension()))
64+
}
65+
66+
pub fn rustc_stage_2(&self) -> Utf8PathBuf {
67+
self.build_artifacts()
68+
.join("stage2")
69+
.join("bin")
70+
.join(format!("rustc{}", executable_extension()))
71+
}
72+
73+
pub fn prebuilt_rustc_perf(&self) -> Option<Utf8PathBuf> {
74+
self.prebuilt_rustc_perf.clone()
75+
}
76+
77+
/// Path to the built rustc-perf benchmark suite.
78+
pub fn rustc_perf_dir(&self) -> Utf8PathBuf {
79+
self.artifact_dir.join("rustc-perf")
80+
}
81+
82+
pub fn host_llvm_dir(&self) -> Utf8PathBuf {
83+
self.host_llvm_dir.clone()
84+
}
85+
86+
pub fn use_bolt(&self) -> bool {
87+
self.use_bolt
88+
}
89+
90+
pub fn supports_shared_llvm(&self) -> bool {
91+
self.shared_llvm
92+
}
93+
94+
pub fn skipped_tests(&self) -> &[String] {
95+
&self.skipped_tests
96+
}
97+
}
98+
99+
/// What is the extension of binary executables on this platform?
100+
#[cfg(target_family = "unix")]
101+
pub fn executable_extension() -> &'static str {
102+
""
103+
}
104+
105+
#[cfg(target_family = "windows")]
106+
pub fn executable_extension() -> &'static str {
107+
".exe"
108+
}

Diff for: ‎src/tools/opt-dist/src/environment/linux.rs

-58
This file was deleted.

0 commit comments

Comments
 (0)