From 1c1febc59db038876d7fe78a1f056bf324fdff6a Mon Sep 17 00:00:00 2001 From: onur-ozkan Date: Tue, 1 Apr 2025 11:19:57 +0300 Subject: [PATCH 01/10] add new config option: `include` Signed-off-by: onur-ozkan --- src/bootstrap/src/core/config/config.rs | 25 ++++++++++++++++++++++- src/bootstrap/src/utils/change_tracker.rs | 5 +++++ 2 files changed, 29 insertions(+), 1 deletion(-) diff --git a/src/bootstrap/src/core/config/config.rs b/src/bootstrap/src/core/config/config.rs index 25ec64f90b53d..2e88108129010 100644 --- a/src/bootstrap/src/core/config/config.rs +++ b/src/bootstrap/src/core/config/config.rs @@ -701,6 +701,7 @@ pub(crate) struct TomlConfig { target: Option>, dist: Option, profile: Option, + include: Option>, } /// This enum is used for deserializing change IDs from TOML, allowing both numeric values and the string `"ignore"`. @@ -753,7 +754,7 @@ trait Merge { impl Merge for TomlConfig { fn merge( &mut self, - TomlConfig { build, install, llvm, gcc, rust, dist, target, profile, change_id }: Self, + TomlConfig { build, install, llvm, gcc, rust, dist, target, profile, change_id, include }: Self, replace: ReplaceOpt, ) { fn do_merge(x: &mut Option, y: Option, replace: ReplaceOpt) { @@ -766,6 +767,17 @@ impl Merge for TomlConfig { } } + for include_path in include.clone().unwrap_or_default() { + let included_toml = Config::get_toml(&include_path).unwrap_or_else(|e| { + eprintln!( + "ERROR: Failed to parse default config profile at '{}': {e}", + include_path.display() + ); + exit!(2); + }); + self.merge(included_toml, ReplaceOpt::Override); + } + self.change_id.inner.merge(change_id.inner, replace); self.profile.merge(profile, replace); @@ -1600,6 +1612,17 @@ impl Config { toml.merge(included_toml, ReplaceOpt::IgnoreDuplicate); } + for include_path in toml.include.clone().unwrap_or_default() { + let included_toml = get_toml(&include_path).unwrap_or_else(|e| { + eprintln!( + "ERROR: Failed to parse default config profile at '{}': {e}", + include_path.display() + ); + exit!(2); + }); + toml.merge(included_toml, ReplaceOpt::Override); + } + let mut override_toml = TomlConfig::default(); for option in flags.set.iter() { fn get_table(option: &str) -> Result { diff --git a/src/bootstrap/src/utils/change_tracker.rs b/src/bootstrap/src/utils/change_tracker.rs index 48b6f77e8a587..3f1885a425f83 100644 --- a/src/bootstrap/src/utils/change_tracker.rs +++ b/src/bootstrap/src/utils/change_tracker.rs @@ -396,4 +396,9 @@ pub const CONFIG_CHANGE_HISTORY: &[ChangeInfo] = &[ severity: ChangeSeverity::Info, summary: "Added a new option `build.compiletest-use-stage0-libtest` to force `compiletest` to use the stage 0 libtest.", }, + ChangeInfo { + change_id: 138934, + severity: ChangeSeverity::Info, + summary: "Added new option `include` to create config extensions.", + }, ]; From 89e3befe63f35b7556614d85ce63214f62a9a771 Mon Sep 17 00:00:00 2001 From: onur-ozkan Date: Tue, 25 Mar 2025 20:17:32 +0300 Subject: [PATCH 02/10] document config extensions Signed-off-by: onur-ozkan --- .../rustc-dev-guide/src/building/suggested.md | 36 +++++++++++++++++++ 1 file changed, 36 insertions(+) diff --git a/src/doc/rustc-dev-guide/src/building/suggested.md b/src/doc/rustc-dev-guide/src/building/suggested.md index 43ff2ba726f91..5d37a78af8855 100644 --- a/src/doc/rustc-dev-guide/src/building/suggested.md +++ b/src/doc/rustc-dev-guide/src/building/suggested.md @@ -20,6 +20,42 @@ your `.git/hooks` folder as `pre-push` (without the `.sh` extension!). You can also install the hook as a step of running `./x setup`! +## Config extensions + +When working on different tasks, you might need to switch between different bootstrap configurations. +Sometimes you may want to keep an old configuration for future use. But saving raw config values in +random files and manually copying and pasting them can quickly become messy, especially if you have a +long history of different configurations. + +To simplify managing multiple configurations, you can create config extensions. + +For example, you can create a simple config file named `cross.toml`: + +```toml +[build] +build = "x86_64-unknown-linux-gnu" +host = ["i686-unknown-linux-gnu"] +target = ["i686-unknown-linux-gnu"] + + +[llvm] +download-ci-llvm = false + +[target.x86_64-unknown-linux-gnu] +llvm-config = "/path/to/llvm-19/bin/llvm-config" +``` + +Then, include this in your `bootstrap.toml`: + +```toml +include = ["cross.toml"] +``` + +You can also include extensions within extensions recursively. + +**Note:** In the `include` field, the overriding logic follows a right-to-left order. Also, the outer +extension/config always overrides the inner ones. + ## Configuring `rust-analyzer` for `rustc` ### Project-local rust-analyzer setup From 4e80659b32a794513646b93890c9a6eeb103de17 Mon Sep 17 00:00:00 2001 From: onur-ozkan Date: Tue, 1 Apr 2025 11:44:33 +0300 Subject: [PATCH 03/10] implement cyclic inclusion handling Signed-off-by: onur-ozkan --- src/bootstrap/src/core/config/config.rs | 51 ++++++++++++++++++------- 1 file changed, 38 insertions(+), 13 deletions(-) diff --git a/src/bootstrap/src/core/config/config.rs b/src/bootstrap/src/core/config/config.rs index 2e88108129010..86566d0eee178 100644 --- a/src/bootstrap/src/core/config/config.rs +++ b/src/bootstrap/src/core/config/config.rs @@ -6,6 +6,7 @@ use std::cell::{Cell, RefCell}; use std::collections::{BTreeSet, HashMap, HashSet}; use std::fmt::{self, Display}; +use std::hash::Hash; use std::io::IsTerminal; use std::path::{Path, PathBuf, absolute}; use std::process::Command; @@ -748,19 +749,25 @@ enum ReplaceOpt { } trait Merge { - fn merge(&mut self, other: Self, replace: ReplaceOpt); + fn merge( + &mut self, + included_extensions: &mut HashSet, + other: Self, + replace: ReplaceOpt, + ); } impl Merge for TomlConfig { fn merge( &mut self, + included_extensions: &mut HashSet, TomlConfig { build, install, llvm, gcc, rust, dist, target, profile, change_id, include }: Self, replace: ReplaceOpt, ) { fn do_merge(x: &mut Option, y: Option, replace: ReplaceOpt) { if let Some(new) = y { if let Some(original) = x { - original.merge(new, replace); + original.merge(&mut Default::default(), new, replace); } else { *x = Some(new); } @@ -775,11 +782,20 @@ impl Merge for TomlConfig { ); exit!(2); }); - self.merge(included_toml, ReplaceOpt::Override); + + assert!( + included_extensions.insert(include_path.clone()), + "Cyclic inclusion detected: '{}' is being included again before its previous inclusion was fully processed.", + include_path.display() + ); + + self.merge(included_extensions, included_toml, ReplaceOpt::Override); + + included_extensions.remove(&include_path); } - self.change_id.inner.merge(change_id.inner, replace); - self.profile.merge(profile, replace); + self.change_id.inner.merge(&mut Default::default(), change_id.inner, replace); + self.profile.merge(&mut Default::default(), profile, replace); do_merge(&mut self.build, build, replace); do_merge(&mut self.install, install, replace); @@ -794,7 +810,7 @@ impl Merge for TomlConfig { (Some(original_target), Some(new_target)) => { for (triple, new) in new_target { if let Some(original) = original_target.get_mut(&triple) { - original.merge(new, replace); + original.merge(&mut Default::default(), new, replace); } else { original_target.insert(triple, new); } @@ -815,7 +831,7 @@ macro_rules! define_config { } impl Merge for $name { - fn merge(&mut self, other: Self, replace: ReplaceOpt) { + fn merge(&mut self, _included_extensions: &mut HashSet, other: Self, replace: ReplaceOpt) { $( match replace { ReplaceOpt::IgnoreDuplicate => { @@ -915,7 +931,12 @@ macro_rules! define_config { } impl Merge for Option { - fn merge(&mut self, other: Self, replace: ReplaceOpt) { + fn merge( + &mut self, + _included_extensions: &mut HashSet, + other: Self, + replace: ReplaceOpt, + ) { match replace { ReplaceOpt::IgnoreDuplicate => { if self.is_none() { @@ -1609,7 +1630,7 @@ impl Config { ); exit!(2); }); - toml.merge(included_toml, ReplaceOpt::IgnoreDuplicate); + toml.merge(&mut Default::default(), included_toml, ReplaceOpt::IgnoreDuplicate); } for include_path in toml.include.clone().unwrap_or_default() { @@ -1620,7 +1641,7 @@ impl Config { ); exit!(2); }); - toml.merge(included_toml, ReplaceOpt::Override); + toml.merge(&mut Default::default(), included_toml, ReplaceOpt::Override); } let mut override_toml = TomlConfig::default(); @@ -1631,7 +1652,7 @@ impl Config { let mut err = match get_table(option) { Ok(v) => { - override_toml.merge(v, ReplaceOpt::ErrorOnDuplicate); + override_toml.merge(&mut Default::default(), v, ReplaceOpt::ErrorOnDuplicate); continue; } Err(e) => e, @@ -1642,7 +1663,11 @@ impl Config { if !value.contains('"') { match get_table(&format!(r#"{key}="{value}""#)) { Ok(v) => { - override_toml.merge(v, ReplaceOpt::ErrorOnDuplicate); + override_toml.merge( + &mut Default::default(), + v, + ReplaceOpt::ErrorOnDuplicate, + ); continue; } Err(e) => err = e, @@ -1652,7 +1677,7 @@ impl Config { eprintln!("failed to parse override `{option}`: `{err}"); exit!(2) } - toml.merge(override_toml, ReplaceOpt::Override); + toml.merge(&mut Default::default(), override_toml, ReplaceOpt::Override); config.change_id = toml.change_id.inner; From 78cb4538ee203b08441382b5cd64b1989ed5b3b9 Mon Sep 17 00:00:00 2001 From: onur-ozkan Date: Tue, 1 Apr 2025 11:53:32 +0300 Subject: [PATCH 04/10] document `include` in `bootstrap.example.toml` Signed-off-by: onur-ozkan --- bootstrap.example.toml | 5 +++++ src/doc/rustc-dev-guide/src/building/suggested.md | 5 +++-- 2 files changed, 8 insertions(+), 2 deletions(-) diff --git a/bootstrap.example.toml b/bootstrap.example.toml index 0927f648635ce..0a314f65ec229 100644 --- a/bootstrap.example.toml +++ b/bootstrap.example.toml @@ -19,6 +19,11 @@ # Note that this has no default value (x.py uses the defaults in `bootstrap.example.toml`). #profile = +# Inherits configuration values from different configuration files (a.k.a. config extensions). +# Supports absolute paths, and uses the current directory (where the bootstrap was invoked) +# as the base if the given path is not absolute. +#include = [] + # Keeps track of major changes made to this configuration. # # This value also represents ID of the PR that caused major changes. Meaning, diff --git a/src/doc/rustc-dev-guide/src/building/suggested.md b/src/doc/rustc-dev-guide/src/building/suggested.md index 5d37a78af8855..b2e258be079d5 100644 --- a/src/doc/rustc-dev-guide/src/building/suggested.md +++ b/src/doc/rustc-dev-guide/src/building/suggested.md @@ -53,8 +53,9 @@ include = ["cross.toml"] You can also include extensions within extensions recursively. -**Note:** In the `include` field, the overriding logic follows a right-to-left order. Also, the outer -extension/config always overrides the inner ones. +**Note:** In the `include` field, the overriding logic follows a right-to-left order. For example, +in `include = ["a.toml", "b.toml"]`, extension `b.toml` overrides `a.toml`. Also, parent extensions +always overrides the inner ones. ## Configuring `rust-analyzer` for `rustc` From 3f70f197f2d131a1f57912f202072b2447d3e326 Mon Sep 17 00:00:00 2001 From: onur-ozkan Date: Tue, 1 Apr 2025 21:56:58 +0000 Subject: [PATCH 05/10] apply nit notes Signed-off-by: onur-ozkan --- bootstrap.example.toml | 3 + src/bootstrap/src/core/config/config.rs | 95 ++++++++++++++++++------- 2 files changed, 72 insertions(+), 26 deletions(-) diff --git a/bootstrap.example.toml b/bootstrap.example.toml index 0a314f65ec229..72c4492d465d8 100644 --- a/bootstrap.example.toml +++ b/bootstrap.example.toml @@ -22,6 +22,9 @@ # Inherits configuration values from different configuration files (a.k.a. config extensions). # Supports absolute paths, and uses the current directory (where the bootstrap was invoked) # as the base if the given path is not absolute. +# +# The overriding logic follows a right-to-left order. For example, in `include = ["a.toml", "b.toml"]`, +# extension `b.toml` overrides `a.toml`. Also, parent extensions always overrides the inner ones. #include = [] # Keeps track of major changes made to this configuration. diff --git a/src/bootstrap/src/core/config/config.rs b/src/bootstrap/src/core/config/config.rs index 86566d0eee178..e70ac74c7dae2 100644 --- a/src/bootstrap/src/core/config/config.rs +++ b/src/bootstrap/src/core/config/config.rs @@ -751,6 +751,7 @@ enum ReplaceOpt { trait Merge { fn merge( &mut self, + parent_config_path: Option, included_extensions: &mut HashSet, other: Self, replace: ReplaceOpt, @@ -760,6 +761,7 @@ trait Merge { impl Merge for TomlConfig { fn merge( &mut self, + parent_config_path: Option, included_extensions: &mut HashSet, TomlConfig { build, install, llvm, gcc, rust, dist, target, profile, change_id, include }: Self, replace: ReplaceOpt, @@ -767,19 +769,27 @@ impl Merge for TomlConfig { fn do_merge(x: &mut Option, y: Option, replace: ReplaceOpt) { if let Some(new) = y { if let Some(original) = x { - original.merge(&mut Default::default(), new, replace); + original.merge(None, &mut Default::default(), new, replace); } else { *x = Some(new); } } } - for include_path in include.clone().unwrap_or_default() { + let parent_dir = parent_config_path + .as_ref() + .and_then(|p| p.parent().map(ToOwned::to_owned)) + .unwrap_or_default(); + + for include_path in include.clone().unwrap_or_default().iter().rev() { + let include_path = parent_dir.join(include_path); + let include_path = include_path.canonicalize().unwrap_or_else(|e| { + eprintln!("ERROR: Failed to canonicalize '{}' path: {e}", include_path.display()); + exit!(2); + }); + let included_toml = Config::get_toml(&include_path).unwrap_or_else(|e| { - eprintln!( - "ERROR: Failed to parse default config profile at '{}': {e}", - include_path.display() - ); + eprintln!("ERROR: Failed to parse '{}': {e}", include_path.display()); exit!(2); }); @@ -789,13 +799,20 @@ impl Merge for TomlConfig { include_path.display() ); - self.merge(included_extensions, included_toml, ReplaceOpt::Override); + self.merge( + Some(include_path.clone()), + included_extensions, + included_toml, + // Ensures that parent configuration always takes precedence + // over child configurations. + ReplaceOpt::IgnoreDuplicate, + ); included_extensions.remove(&include_path); } - self.change_id.inner.merge(&mut Default::default(), change_id.inner, replace); - self.profile.merge(&mut Default::default(), profile, replace); + self.change_id.inner.merge(None, &mut Default::default(), change_id.inner, replace); + self.profile.merge(None, &mut Default::default(), profile, replace); do_merge(&mut self.build, build, replace); do_merge(&mut self.install, install, replace); @@ -810,7 +827,7 @@ impl Merge for TomlConfig { (Some(original_target), Some(new_target)) => { for (triple, new) in new_target { if let Some(original) = original_target.get_mut(&triple) { - original.merge(&mut Default::default(), new, replace); + original.merge(None, &mut Default::default(), new, replace); } else { original_target.insert(triple, new); } @@ -831,7 +848,13 @@ macro_rules! define_config { } impl Merge for $name { - fn merge(&mut self, _included_extensions: &mut HashSet, other: Self, replace: ReplaceOpt) { + fn merge( + &mut self, + _parent_config_path: Option, + _included_extensions: &mut HashSet, + other: Self, + replace: ReplaceOpt + ) { $( match replace { ReplaceOpt::IgnoreDuplicate => { @@ -933,6 +956,7 @@ macro_rules! define_config { impl Merge for Option { fn merge( &mut self, + _parent_config_path: Option, _included_extensions: &mut HashSet, other: Self, replace: ReplaceOpt, @@ -1581,7 +1605,8 @@ impl Config { // but not if `bootstrap.toml` hasn't been created. let mut toml = if !using_default_path || toml_path.exists() { config.config = Some(if cfg!(not(test)) { - toml_path.canonicalize().unwrap() + toml_path = toml_path.canonicalize().unwrap(); + toml_path.clone() } else { toml_path.clone() }); @@ -1609,6 +1634,24 @@ impl Config { toml.profile = Some("dist".into()); } + // Reverse the list to ensure the last added config extension remains the most dominant. + // For example, given ["a.toml", "b.toml"], "b.toml" should take precedence over "a.toml". + // + // This must be handled before applying the `profile` since `include`s should always take + // precedence over `profile`s. + for include_path in toml.include.clone().unwrap_or_default().iter().rev() { + let included_toml = get_toml(include_path).unwrap_or_else(|e| { + eprintln!("ERROR: Failed to parse '{}': {e}", include_path.display()); + exit!(2); + }); + toml.merge( + Some(toml_path.join(include_path)), + &mut Default::default(), + included_toml, + ReplaceOpt::IgnoreDuplicate, + ); + } + if let Some(include) = &toml.profile { // Allows creating alias for profile names, allowing // profiles to be renamed while maintaining back compatibility @@ -1630,18 +1673,12 @@ impl Config { ); exit!(2); }); - toml.merge(&mut Default::default(), included_toml, ReplaceOpt::IgnoreDuplicate); - } - - for include_path in toml.include.clone().unwrap_or_default() { - let included_toml = get_toml(&include_path).unwrap_or_else(|e| { - eprintln!( - "ERROR: Failed to parse default config profile at '{}': {e}", - include_path.display() - ); - exit!(2); - }); - toml.merge(&mut Default::default(), included_toml, ReplaceOpt::Override); + toml.merge( + Some(include_path), + &mut Default::default(), + included_toml, + ReplaceOpt::IgnoreDuplicate, + ); } let mut override_toml = TomlConfig::default(); @@ -1652,7 +1689,12 @@ impl Config { let mut err = match get_table(option) { Ok(v) => { - override_toml.merge(&mut Default::default(), v, ReplaceOpt::ErrorOnDuplicate); + override_toml.merge( + None, + &mut Default::default(), + v, + ReplaceOpt::ErrorOnDuplicate, + ); continue; } Err(e) => e, @@ -1664,6 +1706,7 @@ impl Config { match get_table(&format!(r#"{key}="{value}""#)) { Ok(v) => { override_toml.merge( + None, &mut Default::default(), v, ReplaceOpt::ErrorOnDuplicate, @@ -1677,7 +1720,7 @@ impl Config { eprintln!("failed to parse override `{option}`: `{err}"); exit!(2) } - toml.merge(&mut Default::default(), override_toml, ReplaceOpt::Override); + toml.merge(None, &mut Default::default(), override_toml, ReplaceOpt::Override); config.change_id = toml.change_id.inner; From 8e6f50bb4d30801d04619a4cd6406b320eb8919f Mon Sep 17 00:00:00 2001 From: onur-ozkan Date: Tue, 15 Apr 2025 10:44:19 +0300 Subject: [PATCH 06/10] fix path and the ordering logic Signed-off-by: onur-ozkan --- src/bootstrap/src/core/config/config.rs | 54 +++++++++++++------------ 1 file changed, 28 insertions(+), 26 deletions(-) diff --git a/src/bootstrap/src/core/config/config.rs b/src/bootstrap/src/core/config/config.rs index e70ac74c7dae2..19ed442c500f9 100644 --- a/src/bootstrap/src/core/config/config.rs +++ b/src/bootstrap/src/core/config/config.rs @@ -776,6 +776,30 @@ impl Merge for TomlConfig { } } + self.change_id.inner.merge(None, &mut Default::default(), change_id.inner, replace); + self.profile.merge(None, &mut Default::default(), profile, replace); + + do_merge(&mut self.build, build, replace); + do_merge(&mut self.install, install, replace); + do_merge(&mut self.llvm, llvm, replace); + do_merge(&mut self.gcc, gcc, replace); + do_merge(&mut self.rust, rust, replace); + do_merge(&mut self.dist, dist, replace); + + match (self.target.as_mut(), target) { + (_, None) => {} + (None, Some(target)) => self.target = Some(target), + (Some(original_target), Some(new_target)) => { + for (triple, new) in new_target { + if let Some(original) = original_target.get_mut(&triple) { + original.merge(None, &mut Default::default(), new, replace); + } else { + original_target.insert(triple, new); + } + } + } + } + let parent_dir = parent_config_path .as_ref() .and_then(|p| p.parent().map(ToOwned::to_owned)) @@ -810,30 +834,6 @@ impl Merge for TomlConfig { included_extensions.remove(&include_path); } - - self.change_id.inner.merge(None, &mut Default::default(), change_id.inner, replace); - self.profile.merge(None, &mut Default::default(), profile, replace); - - do_merge(&mut self.build, build, replace); - do_merge(&mut self.install, install, replace); - do_merge(&mut self.llvm, llvm, replace); - do_merge(&mut self.gcc, gcc, replace); - do_merge(&mut self.rust, rust, replace); - do_merge(&mut self.dist, dist, replace); - - match (self.target.as_mut(), target) { - (_, None) => {} - (None, Some(target)) => self.target = Some(target), - (Some(original_target), Some(new_target)) => { - for (triple, new) in new_target { - if let Some(original) = original_target.get_mut(&triple) { - original.merge(None, &mut Default::default(), new, replace); - } else { - original_target.insert(triple, new); - } - } - } - } } } @@ -1640,12 +1640,14 @@ impl Config { // This must be handled before applying the `profile` since `include`s should always take // precedence over `profile`s. for include_path in toml.include.clone().unwrap_or_default().iter().rev() { - let included_toml = get_toml(include_path).unwrap_or_else(|e| { + let include_path = toml_path.parent().unwrap().join(include_path); + + let included_toml = get_toml(&include_path).unwrap_or_else(|e| { eprintln!("ERROR: Failed to parse '{}': {e}", include_path.display()); exit!(2); }); toml.merge( - Some(toml_path.join(include_path)), + Some(include_path), &mut Default::default(), included_toml, ReplaceOpt::IgnoreDuplicate, From 7dfb457745d7f873eb5db1c3fb39d58fa768740d Mon Sep 17 00:00:00 2001 From: onur-ozkan Date: Tue, 15 Apr 2025 11:53:21 +0300 Subject: [PATCH 07/10] add FIXME note in `TomlConfig::merge` Signed-off-by: onur-ozkan --- src/bootstrap/src/core/config/config.rs | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/bootstrap/src/core/config/config.rs b/src/bootstrap/src/core/config/config.rs index 19ed442c500f9..b7afd81dfdb5b 100644 --- a/src/bootstrap/src/core/config/config.rs +++ b/src/bootstrap/src/core/config/config.rs @@ -812,6 +812,8 @@ impl Merge for TomlConfig { exit!(2); }); + // FIXME: Similar to `Config::parse_inner`, allow passing a custom `get_toml` from the caller to + // improve testability since `Config::get_toml` does nothing when `cfg(test)` is enabled. let included_toml = Config::get_toml(&include_path).unwrap_or_else(|e| { eprintln!("ERROR: Failed to parse '{}': {e}", include_path.display()); exit!(2); From 6d52b51d3e6321b7a1d24e5f995aa709057153e3 Mon Sep 17 00:00:00 2001 From: onur-ozkan Date: Tue, 15 Apr 2025 19:10:31 +0300 Subject: [PATCH 08/10] add comment in `TomlConfig::merge` about the merge order Signed-off-by: onur-ozkan --- src/bootstrap/src/core/config/config.rs | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/bootstrap/src/core/config/config.rs b/src/bootstrap/src/core/config/config.rs index b7afd81dfdb5b..e15aab4ad506e 100644 --- a/src/bootstrap/src/core/config/config.rs +++ b/src/bootstrap/src/core/config/config.rs @@ -805,6 +805,8 @@ impl Merge for TomlConfig { .and_then(|p| p.parent().map(ToOwned::to_owned)) .unwrap_or_default(); + // `include` handled later since we ignore duplicates using `ReplaceOpt::IgnoreDuplicate` to + // keep the upper-level configuration to take precedence. for include_path in include.clone().unwrap_or_default().iter().rev() { let include_path = parent_dir.join(include_path); let include_path = include_path.canonicalize().unwrap_or_else(|e| { From 827047895a4c036f652b7105443f6305164f0e25 Mon Sep 17 00:00:00 2001 From: onur-ozkan Date: Wed, 16 Apr 2025 20:53:51 +0300 Subject: [PATCH 09/10] resolve config include FIXME Signed-off-by: onur-ozkan --- src/bootstrap/src/core/config/config.rs | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/src/bootstrap/src/core/config/config.rs b/src/bootstrap/src/core/config/config.rs index e15aab4ad506e..0c221abc9698d 100644 --- a/src/bootstrap/src/core/config/config.rs +++ b/src/bootstrap/src/core/config/config.rs @@ -814,9 +814,7 @@ impl Merge for TomlConfig { exit!(2); }); - // FIXME: Similar to `Config::parse_inner`, allow passing a custom `get_toml` from the caller to - // improve testability since `Config::get_toml` does nothing when `cfg(test)` is enabled. - let included_toml = Config::get_toml(&include_path).unwrap_or_else(|e| { + let included_toml = Config::get_toml_inner(&include_path).unwrap_or_else(|e| { eprintln!("ERROR: Failed to parse '{}': {e}", include_path.display()); exit!(2); }); @@ -1424,13 +1422,15 @@ impl Config { Self::get_toml(&builder_config_path) } - #[cfg(test)] - pub(crate) fn get_toml(_: &Path) -> Result { - Ok(TomlConfig::default()) + pub(crate) fn get_toml(file: &Path) -> Result { + #[cfg(test)] + return Ok(TomlConfig::default()); + + #[cfg(not(test))] + Self::get_toml_inner(file) } - #[cfg(not(test))] - pub(crate) fn get_toml(file: &Path) -> Result { + fn get_toml_inner(file: &Path) -> Result { let contents = t!(fs::read_to_string(file), format!("config file {} not found", file.display())); // Deserialize to Value and then TomlConfig to prevent the Deserialize impl of From ac7d1be031ac08cc0910efa3ec2f4186d7943792 Mon Sep 17 00:00:00 2001 From: onur-ozkan Date: Wed, 16 Apr 2025 20:54:15 +0300 Subject: [PATCH 10/10] add coverage on config include logic Signed-off-by: onur-ozkan --- src/bootstrap/src/core/config/tests.rs | 211 ++++++++++++++++++++++++- 1 file changed, 209 insertions(+), 2 deletions(-) diff --git a/src/bootstrap/src/core/config/tests.rs b/src/bootstrap/src/core/config/tests.rs index d8002ba8467bd..c8a12c9072c91 100644 --- a/src/bootstrap/src/core/config/tests.rs +++ b/src/bootstrap/src/core/config/tests.rs @@ -1,8 +1,8 @@ use std::collections::BTreeSet; -use std::env; use std::fs::{File, remove_file}; use std::io::Write; -use std::path::Path; +use std::path::{Path, PathBuf}; +use std::{env, fs}; use build_helper::ci::CiEnv; use clap::CommandFactory; @@ -23,6 +23,27 @@ pub(crate) fn parse(config: &str) -> Config { ) } +fn get_toml(file: &Path) -> Result { + let contents = std::fs::read_to_string(file).unwrap(); + toml::from_str(&contents).and_then(|table: toml::Value| TomlConfig::deserialize(table)) +} + +/// Helps with debugging by using consistent test-specific directories instead of +/// random temporary directories. +fn prepare_test_specific_dir() -> PathBuf { + let current = std::thread::current(); + // Replace "::" with "_" to make it safe for directory names on Windows systems + let test_path = current.name().unwrap().replace("::", "_"); + + let testdir = parse("").tempdir().join(test_path); + + // clean up any old test files + let _ = fs::remove_dir_all(&testdir); + let _ = fs::create_dir_all(&testdir); + + testdir +} + #[test] fn download_ci_llvm() { let config = parse("llvm.download-ci-llvm = false"); @@ -539,3 +560,189 @@ fn test_ci_flag() { let config = Config::parse_inner(Flags::parse(&["check".into()]), |&_| toml::from_str("")); assert_eq!(config.is_running_on_ci, CiEnv::is_ci()); } + +#[test] +fn test_precedence_of_includes() { + let testdir = prepare_test_specific_dir(); + + let root_config = testdir.join("config.toml"); + let root_config_content = br#" + include = ["./extension.toml"] + + [llvm] + link-jobs = 2 + "#; + File::create(&root_config).unwrap().write_all(root_config_content).unwrap(); + + let extension = testdir.join("extension.toml"); + let extension_content = br#" + change-id=543 + include = ["./extension2.toml"] + "#; + File::create(extension).unwrap().write_all(extension_content).unwrap(); + + let extension = testdir.join("extension2.toml"); + let extension_content = br#" + change-id=742 + + [llvm] + link-jobs = 10 + + [build] + description = "Some creative description" + "#; + File::create(extension).unwrap().write_all(extension_content).unwrap(); + + let config = Config::parse_inner( + Flags::parse(&["check".to_owned(), format!("--config={}", root_config.to_str().unwrap())]), + get_toml, + ); + + assert_eq!(config.change_id.unwrap(), ChangeId::Id(543)); + assert_eq!(config.llvm_link_jobs.unwrap(), 2); + assert_eq!(config.description.unwrap(), "Some creative description"); +} + +#[test] +#[should_panic(expected = "Cyclic inclusion detected")] +fn test_cyclic_include_direct() { + let testdir = prepare_test_specific_dir(); + + let root_config = testdir.join("config.toml"); + let root_config_content = br#" + include = ["./extension.toml"] + "#; + File::create(&root_config).unwrap().write_all(root_config_content).unwrap(); + + let extension = testdir.join("extension.toml"); + let extension_content = br#" + include = ["./config.toml"] + "#; + File::create(extension).unwrap().write_all(extension_content).unwrap(); + + let config = Config::parse_inner( + Flags::parse(&["check".to_owned(), format!("--config={}", root_config.to_str().unwrap())]), + get_toml, + ); +} + +#[test] +#[should_panic(expected = "Cyclic inclusion detected")] +fn test_cyclic_include_indirect() { + let testdir = prepare_test_specific_dir(); + + let root_config = testdir.join("config.toml"); + let root_config_content = br#" + include = ["./extension.toml"] + "#; + File::create(&root_config).unwrap().write_all(root_config_content).unwrap(); + + let extension = testdir.join("extension.toml"); + let extension_content = br#" + include = ["./extension2.toml"] + "#; + File::create(extension).unwrap().write_all(extension_content).unwrap(); + + let extension = testdir.join("extension2.toml"); + let extension_content = br#" + include = ["./extension3.toml"] + "#; + File::create(extension).unwrap().write_all(extension_content).unwrap(); + + let extension = testdir.join("extension3.toml"); + let extension_content = br#" + include = ["./extension.toml"] + "#; + File::create(extension).unwrap().write_all(extension_content).unwrap(); + + let config = Config::parse_inner( + Flags::parse(&["check".to_owned(), format!("--config={}", root_config.to_str().unwrap())]), + get_toml, + ); +} + +#[test] +fn test_include_absolute_paths() { + let testdir = prepare_test_specific_dir(); + + let extension = testdir.join("extension.toml"); + File::create(&extension).unwrap().write_all(&[]).unwrap(); + + let root_config = testdir.join("config.toml"); + let extension_absolute_path = + extension.canonicalize().unwrap().to_str().unwrap().replace('\\', r"\\"); + let root_config_content = format!(r#"include = ["{}"]"#, extension_absolute_path); + File::create(&root_config).unwrap().write_all(root_config_content.as_bytes()).unwrap(); + + let config = Config::parse_inner( + Flags::parse(&["check".to_owned(), format!("--config={}", root_config.to_str().unwrap())]), + get_toml, + ); +} + +#[test] +fn test_include_relative_paths() { + let testdir = prepare_test_specific_dir(); + + let _ = fs::create_dir_all(&testdir.join("subdir/another_subdir")); + + let root_config = testdir.join("config.toml"); + let root_config_content = br#" + include = ["./subdir/extension.toml"] + "#; + File::create(&root_config).unwrap().write_all(root_config_content).unwrap(); + + let extension = testdir.join("subdir/extension.toml"); + let extension_content = br#" + include = ["../extension2.toml"] + "#; + File::create(extension).unwrap().write_all(extension_content).unwrap(); + + let extension = testdir.join("extension2.toml"); + let extension_content = br#" + include = ["./subdir/another_subdir/extension3.toml"] + "#; + File::create(extension).unwrap().write_all(extension_content).unwrap(); + + let extension = testdir.join("subdir/another_subdir/extension3.toml"); + let extension_content = br#" + include = ["../../extension4.toml"] + "#; + File::create(extension).unwrap().write_all(extension_content).unwrap(); + + let extension = testdir.join("extension4.toml"); + File::create(extension).unwrap().write_all(&[]).unwrap(); + + let config = Config::parse_inner( + Flags::parse(&["check".to_owned(), format!("--config={}", root_config.to_str().unwrap())]), + get_toml, + ); +} + +#[test] +fn test_include_precedence_over_profile() { + let testdir = prepare_test_specific_dir(); + + let root_config = testdir.join("config.toml"); + let root_config_content = br#" + profile = "dist" + include = ["./extension.toml"] + "#; + File::create(&root_config).unwrap().write_all(root_config_content).unwrap(); + + let extension = testdir.join("extension.toml"); + let extension_content = br#" + [rust] + channel = "dev" + "#; + File::create(extension).unwrap().write_all(extension_content).unwrap(); + + let config = Config::parse_inner( + Flags::parse(&["check".to_owned(), format!("--config={}", root_config.to_str().unwrap())]), + get_toml, + ); + + // "dist" profile would normally set the channel to "auto-detect", but includes should + // override profile settings, so we expect this to be "dev" here. + assert_eq!(config.channel, "dev"); +}