From d14df2652df492fe0f19c5e0ae5041b39b5a4d20 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jakub=20Ber=C3=A1nek?= Date: Wed, 16 Apr 2025 13:24:01 +0200 Subject: [PATCH 01/12] Make `parent` in `download_auto_job_metrics` optional --- src/ci/citool/src/main.rs | 2 +- src/ci/citool/src/metrics.rs | 23 ++++++++++++----------- 2 files changed, 13 insertions(+), 12 deletions(-) diff --git a/src/ci/citool/src/main.rs b/src/ci/citool/src/main.rs index a1956da352f5c..0fee862f5721b 100644 --- a/src/ci/citool/src/main.rs +++ b/src/ci/citool/src/main.rs @@ -180,7 +180,7 @@ fn postprocess_metrics( } fn post_merge_report(db: JobDatabase, current: String, parent: String) -> anyhow::Result<()> { - let metrics = download_auto_job_metrics(&db, &parent, ¤t)?; + let metrics = download_auto_job_metrics(&db, Some(&parent), ¤t)?; println!("\nComparing {parent} (parent) -> {current} (this PR)\n"); diff --git a/src/ci/citool/src/metrics.rs b/src/ci/citool/src/metrics.rs index a816fb3c4f165..3d8b1ad84cf72 100644 --- a/src/ci/citool/src/metrics.rs +++ b/src/ci/citool/src/metrics.rs @@ -46,24 +46,25 @@ pub struct JobMetrics { /// `parent` and `current` should be commit SHAs. pub fn download_auto_job_metrics( job_db: &JobDatabase, - parent: &str, + parent: Option<&str>, current: &str, ) -> anyhow::Result> { let mut jobs = HashMap::default(); for job in &job_db.auto_jobs { eprintln!("Downloading metrics of job {}", job.name); - let metrics_parent = match download_job_metrics(&job.name, parent) { - Ok(metrics) => Some(metrics), - Err(error) => { - eprintln!( - r#"Did not find metrics for job `{}` at `{parent}`: {error:?}. + let metrics_parent = + parent.and_then(|parent| match download_job_metrics(&job.name, parent) { + Ok(metrics) => Some(metrics), + Err(error) => { + eprintln!( + r#"Did not find metrics for job `{}` at `{parent}`: {error:?}. Maybe it was newly added?"#, - job.name - ); - None - } - }; + job.name + ); + None + } + }); let metrics_current = download_job_metrics(&job.name, current)?; jobs.insert( job.name.clone(), From 111c15c48e618b30a0cf11d91b135d87d73053a4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jakub=20Ber=C3=A1nek?= Date: Wed, 16 Apr 2025 17:20:53 +0200 Subject: [PATCH 02/12] Extract function for normalizing path delimiters to `utils` --- src/ci/citool/src/analysis.rs | 13 ++++++------- src/ci/citool/src/utils.rs | 6 ++++++ 2 files changed, 12 insertions(+), 7 deletions(-) diff --git a/src/ci/citool/src/analysis.rs b/src/ci/citool/src/analysis.rs index 9fc7c309bfbdc..62974be2dbe8c 100644 --- a/src/ci/citool/src/analysis.rs +++ b/src/ci/citool/src/analysis.rs @@ -8,9 +8,9 @@ use build_helper::metrics::{ }; use crate::github::JobInfoResolver; -use crate::metrics; use crate::metrics::{JobMetrics, JobName, get_test_suites}; use crate::utils::{output_details, pluralize}; +use crate::{metrics, utils}; /// Outputs durations of individual bootstrap steps from the gathered bootstrap invocations, /// and also a table with summarized information about executed tests. @@ -394,18 +394,17 @@ fn aggregate_tests(metrics: &JsonRoot) -> TestSuiteData { // Poor man's detection of doctests based on the "(line XYZ)" suffix let is_doctest = matches!(suite.metadata, TestSuiteMetadata::CargoPackage { .. }) && test.name.contains("(line"); - let test_entry = Test { name: generate_test_name(&test.name), stage, is_doctest }; + let test_entry = Test { + name: utils::normalize_path_delimiters(&test.name).to_string(), + stage, + is_doctest, + }; tests.insert(test_entry, test.outcome.clone()); } } TestSuiteData { tests } } -/// Normalizes Windows-style path delimiters to Unix-style paths. -fn generate_test_name(name: &str) -> String { - name.replace('\\', "/") -} - /// Prints test changes in Markdown format to stdout. fn report_test_diffs( diff: AggregatedTestDiffs, diff --git a/src/ci/citool/src/utils.rs b/src/ci/citool/src/utils.rs index a4c6ff85ef73c..0367d349a1ef4 100644 --- a/src/ci/citool/src/utils.rs +++ b/src/ci/citool/src/utils.rs @@ -1,3 +1,4 @@ +use std::borrow::Cow; use std::path::Path; use anyhow::Context; @@ -28,3 +29,8 @@ where func(); println!("\n"); } + +/// Normalizes Windows-style path delimiters to Unix-style paths. +pub fn normalize_path_delimiters(name: &str) -> Cow { + if name.contains("\\") { name.replace('\\', "/").into() } else { name.into() } +} From c8a882b7b58a7b8f6b276ab64117b55bbe2626e7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jakub=20Ber=C3=A1nek?= Date: Thu, 17 Apr 2025 09:41:12 +0200 Subject: [PATCH 03/12] Add command to `citool` for generating a test dashboard --- src/ci/citool/Cargo.lock | 67 ++++++ src/ci/citool/Cargo.toml | 1 + src/ci/citool/src/main.rs | 16 +- src/ci/citool/src/test_dashboard/mod.rs | 239 +++++++++++++++++++++ src/ci/citool/templates/layout.askama | 71 ++++++ src/ci/citool/templates/test_group.askama | 22 ++ src/ci/citool/templates/test_suites.askama | 18 ++ 7 files changed, 433 insertions(+), 1 deletion(-) create mode 100644 src/ci/citool/src/test_dashboard/mod.rs create mode 100644 src/ci/citool/templates/layout.askama create mode 100644 src/ci/citool/templates/test_group.askama create mode 100644 src/ci/citool/templates/test_suites.askama diff --git a/src/ci/citool/Cargo.lock b/src/ci/citool/Cargo.lock index 2fe219f368b9c..43321d12cafcd 100644 --- a/src/ci/citool/Cargo.lock +++ b/src/ci/citool/Cargo.lock @@ -64,12 +64,63 @@ version = "1.0.95" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "34ac096ce696dc2fcabef30516bb13c0a68a11d30131d3df6f04711467681b04" +[[package]] +name = "askama" +version = "0.13.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5d4744ed2eef2645831b441d8f5459689ade2ab27c854488fbab1fbe94fce1a7" +dependencies = [ + "askama_derive", + "itoa", + "percent-encoding", + "serde", + "serde_json", +] + +[[package]] +name = "askama_derive" +version = "0.13.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d661e0f57be36a5c14c48f78d09011e67e0cb618f269cca9f2fd8d15b68c46ac" +dependencies = [ + "askama_parser", + "basic-toml", + "memchr", + "proc-macro2", + "quote", + "rustc-hash", + "serde", + "serde_derive", + "syn", +] + +[[package]] +name = "askama_parser" +version = "0.13.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cf315ce6524c857bb129ff794935cf6d42c82a6cff60526fe2a63593de4d0d4f" +dependencies = [ + "memchr", + "serde", + "serde_derive", + "winnow", +] + [[package]] name = "base64" version = "0.22.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "72b3254f16251a8381aa12e40e3c4d2f0199f8c6508fbecb9d91f575e0fbb8c6" +[[package]] +name = "basic-toml" +version = "0.1.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ba62675e8242a4c4e806d12f11d136e626e6c8361d6b829310732241652a178a" +dependencies = [ + "serde", +] + [[package]] name = "build_helper" version = "0.1.0" @@ -104,6 +155,7 @@ name = "citool" version = "0.1.0" dependencies = [ "anyhow", + "askama", "build_helper", "clap", "csv", @@ -646,6 +698,12 @@ dependencies = [ "windows-sys 0.52.0", ] +[[package]] +name = "rustc-hash" +version = "2.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "357703d41365b4b27c590e3ed91eabb1b663f07c4c084095e60cbed4362dff0d" + [[package]] name = "rustls" version = "0.23.23" @@ -1026,6 +1084,15 @@ version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "589f6da84c646204747d1270a2a5661ea66ed1cced2631d546fdfb155959f9ec" +[[package]] +name = "winnow" +version = "0.7.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "63d3fcd9bba44b03821e7d699eeee959f3126dcc4aa8e4ae18ec617c2a5cea10" +dependencies = [ + "memchr", +] + [[package]] name = "write16" version = "1.0.0" diff --git a/src/ci/citool/Cargo.toml b/src/ci/citool/Cargo.toml index f18436a126359..0e2aba3b9e3fc 100644 --- a/src/ci/citool/Cargo.toml +++ b/src/ci/citool/Cargo.toml @@ -5,6 +5,7 @@ edition = "2021" [dependencies] anyhow = "1" +askama = "0.13" clap = { version = "4.5", features = ["derive"] } csv = "1" diff = "0.1" diff --git a/src/ci/citool/src/main.rs b/src/ci/citool/src/main.rs index 0fee862f5721b..a7a289fc3d4b1 100644 --- a/src/ci/citool/src/main.rs +++ b/src/ci/citool/src/main.rs @@ -4,6 +4,7 @@ mod datadog; mod github; mod jobs; mod metrics; +mod test_dashboard; mod utils; use std::collections::{BTreeMap, HashMap}; @@ -22,6 +23,7 @@ use crate::datadog::upload_datadog_metric; use crate::github::JobInfoResolver; use crate::jobs::RunType; use crate::metrics::{JobMetrics, download_auto_job_metrics, download_job_metrics, load_metrics}; +use crate::test_dashboard::generate_test_dashboard; use crate::utils::load_env_var; const CI_DIRECTORY: &str = concat!(env!("CARGO_MANIFEST_DIR"), "/.."); @@ -234,6 +236,14 @@ enum Args { /// Current commit that will be compared to `parent`. current: String, }, + /// Generate a directory containing a HTML dashboard of test results from a CI run. + TestDashboard { + /// Commit SHA that was tested on CI to analyze. + current: String, + /// Output path for the HTML directory. + #[clap(long)] + output_dir: PathBuf, + }, } #[derive(clap::ValueEnum, Clone)] @@ -275,7 +285,11 @@ fn main() -> anyhow::Result<()> { postprocess_metrics(metrics_path, parent, job_name)?; } Args::PostMergeReport { current, parent } => { - post_merge_report(load_db(default_jobs_file)?, current, parent)?; + post_merge_report(load_db(&default_jobs_file)?, current, parent)?; + } + Args::TestDashboard { current, output_dir } => { + let db = load_db(&default_jobs_file)?; + generate_test_dashboard(db, ¤t, &output_dir)?; } } diff --git a/src/ci/citool/src/test_dashboard/mod.rs b/src/ci/citool/src/test_dashboard/mod.rs new file mode 100644 index 0000000000000..ad9fe029e15df --- /dev/null +++ b/src/ci/citool/src/test_dashboard/mod.rs @@ -0,0 +1,239 @@ +use std::collections::{BTreeMap, HashMap}; +use std::fs::File; +use std::io::BufWriter; +use std::path::{Path, PathBuf}; + +use askama::Template; +use build_helper::metrics::{TestOutcome, TestSuiteMetadata}; + +use crate::jobs::JobDatabase; +use crate::metrics::{JobMetrics, JobName, download_auto_job_metrics, get_test_suites}; +use crate::utils::normalize_path_delimiters; + +pub struct TestInfo { + name: String, + jobs: Vec, +} + +struct JobTestResult { + job_name: String, + outcome: TestOutcome, +} + +#[derive(Default)] +struct TestSuiteInfo { + name: String, + tests: BTreeMap, +} + +/// Generate a set of HTML files into a directory that contain a dashboard of test results. +pub fn generate_test_dashboard( + db: JobDatabase, + current: &str, + output_dir: &Path, +) -> anyhow::Result<()> { + let metrics = download_auto_job_metrics(&db, None, current)?; + + let suites = gather_test_suites(&metrics); + + std::fs::create_dir_all(output_dir)?; + + let test_count = suites.test_count(); + write_page(output_dir, "index.html", &TestSuitesPage { suites, test_count })?; + + Ok(()) +} + +fn write_page(dir: &Path, name: &str, template: &T) -> anyhow::Result<()> { + let mut file = BufWriter::new(File::create(dir.join(name))?); + Template::write_into(template, &mut file)?; + Ok(()) +} + +fn gather_test_suites(job_metrics: &HashMap) -> TestSuites { + struct CoarseTestSuite<'a> { + kind: TestSuiteKind, + tests: BTreeMap>, + } + + let mut suites: HashMap = HashMap::new(); + + // First, gather tests from all jobs, stages and targets, and aggregate them per suite + for (job, metrics) in job_metrics { + let test_suites = get_test_suites(&metrics.current); + for suite in test_suites { + let (suite_name, stage, target, kind) = match &suite.metadata { + TestSuiteMetadata::CargoPackage { crates, stage, target, .. } => { + (crates.join(","), *stage, target, TestSuiteKind::Cargo) + } + TestSuiteMetadata::Compiletest { suite, stage, target, .. } => { + (suite.clone(), *stage, target, TestSuiteKind::Compiletest) + } + }; + let suite_entry = suites + .entry(suite_name.clone()) + .or_insert_with(|| CoarseTestSuite { kind, tests: Default::default() }); + let test_metadata = TestMetadata { job, stage, target }; + + for test in &suite.tests { + let test_name = normalize_test_name(&test.name, &suite_name); + let test_entry = suite_entry + .tests + .entry(test_name.clone()) + .or_insert_with(|| Test { name: test_name, passed: vec![], ignored: vec![] }); + match test.outcome { + TestOutcome::Passed => { + test_entry.passed.push(test_metadata); + } + TestOutcome::Ignored { ignore_reason: _ } => { + test_entry.ignored.push(test_metadata); + } + TestOutcome::Failed => { + eprintln!("Warning: failed test"); + } + } + } + } + } + + // Then, split the suites per directory + let mut suites = suites.into_iter().collect::>(); + suites.sort_by(|a, b| a.1.kind.cmp(&b.1.kind).then_with(|| a.0.cmp(&b.0))); + + let mut target_suites = vec![]; + for (suite_name, suite) in suites { + let suite = match suite.kind { + TestSuiteKind::Compiletest => TestSuite { + name: suite_name.clone(), + kind: TestSuiteKind::Compiletest, + group: build_test_group(&suite_name, suite.tests), + }, + TestSuiteKind::Cargo => { + let mut tests: Vec<_> = suite.tests.into_iter().collect(); + tests.sort_by(|a, b| a.0.cmp(&b.0)); + TestSuite { + name: format!("[cargo] {}", suite_name.clone()), + kind: TestSuiteKind::Cargo, + group: TestGroup { + name: suite_name, + root_tests: tests.into_iter().map(|t| t.1).collect(), + groups: vec![], + }, + } + } + }; + target_suites.push(suite); + } + + TestSuites { suites: target_suites } +} + +/// Recursively expand a test group based on filesystem hierarchy. +fn build_test_group<'a>(name: &str, tests: BTreeMap>) -> TestGroup<'a> { + let mut root_tests = vec![]; + let mut subdirs: BTreeMap>> = Default::default(); + + // Split tests into root tests and tests located in subdirectories + for (name, test) in tests { + let mut components = Path::new(&name).components().peekable(); + let subdir = components.next().unwrap(); + + if components.peek().is_none() { + // This is a root test + root_tests.push(test); + } else { + // This is a test in a nested directory + let subdir_tests = + subdirs.entry(subdir.as_os_str().to_str().unwrap().to_string()).or_default(); + let test_name = + components.into_iter().collect::().to_str().unwrap().to_string(); + subdir_tests.insert(test_name, test); + } + } + let dirs = subdirs + .into_iter() + .map(|(name, tests)| { + let group = build_test_group(&name, tests); + (name, group) + }) + .collect(); + + TestGroup { name: name.to_string(), root_tests, groups: dirs } +} + +/// Compiletest tests start with `[suite] tests/[suite]/a/b/c...`. +/// Remove the `[suite] tests/[suite]/` prefix so that we can find the filesystem path. +/// Also normalizes path delimiters. +fn normalize_test_name(name: &str, suite_name: &str) -> String { + let name = normalize_path_delimiters(name); + let name = name.as_ref(); + let name = name.strip_prefix(&format!("[{suite_name}]")).unwrap_or(name).trim(); + let name = name.strip_prefix("tests/").unwrap_or(name); + let name = name.strip_prefix(suite_name).unwrap_or(name); + name.trim_start_matches("/").to_string() +} + +#[derive(serde::Serialize)] +struct TestSuites<'a> { + suites: Vec>, +} + +impl<'a> TestSuites<'a> { + fn test_count(&self) -> u64 { + self.suites.iter().map(|suite| suite.group.test_count()).sum::() + } +} + +#[derive(serde::Serialize)] +struct TestSuite<'a> { + name: String, + kind: TestSuiteKind, + group: TestGroup<'a>, +} + +#[derive(Debug, serde::Serialize)] +struct Test<'a> { + name: String, + passed: Vec>, + ignored: Vec>, +} + +#[derive(Clone, Copy, Debug, serde::Serialize)] +struct TestMetadata<'a> { + job: &'a str, + stage: u32, + target: &'a str, +} + +// We have to use a template for the TestGroup instead of a macro, because +// macros cannot be recursive in askama at the moment. +#[derive(Template, serde::Serialize)] +#[template(path = "test_group.askama")] +/// Represents a group of tests +struct TestGroup<'a> { + name: String, + /// Tests located directly in this directory + root_tests: Vec>, + /// Nested directories with additional tests + groups: Vec<(String, TestGroup<'a>)>, +} + +impl<'a> TestGroup<'a> { + fn test_count(&self) -> u64 { + let root = self.root_tests.len() as u64; + self.groups.iter().map(|(_, group)| group.test_count()).sum::() + root + } +} + +#[derive(PartialEq, Eq, PartialOrd, Ord, serde::Serialize)] +enum TestSuiteKind { + Compiletest, + Cargo, +} + +#[derive(Template)] +#[template(path = "test_suites.askama")] +struct TestSuitesPage<'a> { + suites: TestSuites<'a>, + test_count: u64, +} diff --git a/src/ci/citool/templates/layout.askama b/src/ci/citool/templates/layout.askama new file mode 100644 index 0000000000000..2e830aaa9f583 --- /dev/null +++ b/src/ci/citool/templates/layout.askama @@ -0,0 +1,71 @@ + + + + Rust CI Test Dashboard + + + + +{% block content %}{% endblock %} + + diff --git a/src/ci/citool/templates/test_group.askama b/src/ci/citool/templates/test_group.askama new file mode 100644 index 0000000000000..a0b7fa863e577 --- /dev/null +++ b/src/ci/citool/templates/test_group.askama @@ -0,0 +1,22 @@ +
  • +
    +{{ name }} ({{ test_count() }} test{{ test_count() | pluralize }}) + +{% if !groups.is_empty() %} +
      + {% for (dir_name, subgroup) in groups %} + {{ subgroup|safe }} + {% endfor %} +
    +{% endif %} + +{% if !root_tests.is_empty() %} +
      + {% for test in root_tests %} +
    • {{ test.name }} ({{ test.passed.len() }} passed, {{ test.ignored.len() }} ignored)
      • + {% endfor %} +
    +{% endif %} + +
    +
    • diff --git a/src/ci/citool/templates/test_suites.askama b/src/ci/citool/templates/test_suites.askama new file mode 100644 index 0000000000000..a6f8d0e1abe03 --- /dev/null +++ b/src/ci/citool/templates/test_suites.askama @@ -0,0 +1,18 @@ +{% extends "layout.askama" %} + +{% block content %} +

    Rust CI Test Dashboard

    +
    +
    + Total tests: {{ test_count }} +
    + +
      + {% for suite in suites.suites %} + {% if suite.kind == TestSuiteKind::Compiletest %} + {{ suite.group|safe }} + {% endif %} + {% endfor %} +
    +
    +{% endblock %} From a326afd5dd810427c72ed81e705c0d903e74edcb Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jakub=20Ber=C3=A1nek?= Date: Thu, 17 Apr 2025 16:30:23 +0200 Subject: [PATCH 04/12] Add buttons for expanding and collapsing all test suites --- src/ci/citool/templates/layout.askama | 57 ++------------- src/ci/citool/templates/test_suites.askama | 81 +++++++++++++++++++++- 2 files changed, 84 insertions(+), 54 deletions(-) diff --git a/src/ci/citool/templates/layout.askama b/src/ci/citool/templates/layout.askama index 2e830aaa9f583..3b3b6f23741d4 100644 --- a/src/ci/citool/templates/layout.askama +++ b/src/ci/citool/templates/layout.askama @@ -3,69 +3,20 @@ Rust CI Test Dashboard {% block content %}{% endblock %} +{% block scripts %}{% endblock %} diff --git a/src/ci/citool/templates/test_suites.askama b/src/ci/citool/templates/test_suites.askama index a6f8d0e1abe03..a8cedc65f2434 100644 --- a/src/ci/citool/templates/test_suites.askama +++ b/src/ci/citool/templates/test_suites.askama @@ -4,7 +4,11 @@

    Rust CI Test Dashboard

    - Total tests: {{ test_count }} + Total tests: {{ test_count }} +
    + + +
      @@ -16,3 +20,78 @@
    {% endblock %} + +{% block styles %} +h1 { + text-align: center; + color: #333333; + margin-bottom: 30px; +} + +.summary { + display: flex; + justify-content: space-between; +} + +.test-suites { + background: white; + border-radius: 8px; + box-shadow: 0 2px 4px rgba(0, 0, 0, 0.1); + padding: 20px; +} + +ul { + padding-left: 0; +} + +li { + list-style: none; + padding-left: 20px; +} +summary { + margin-bottom: 5px; + padding: 6px; + background-color: #F4F4F4; + border: 1px solid #ddd; + border-radius: 4px; + cursor: pointer; +} +summary:hover { + background-color: #CFCFCF; +} + +/* Style the disclosure triangles */ +details > summary { + list-style: none; + position: relative; +} + +details > summary::before { + content: "▶"; + position: absolute; + left: -15px; + transform: rotate(0); + transition: transform 0.2s; +} + +details[open] > summary::before { + transform: rotate(90deg); +} +{% endblock %} + +{% block scripts %} + +{% endblock %} From 4b310338f8d2a67cbc863ee799206709e95da6b1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jakub=20Ber=C3=A1nek?= Date: Thu, 17 Apr 2025 16:35:34 +0200 Subject: [PATCH 05/12] Add a note about how to find tests that haven't been executed anywhere. --- src/ci/citool/src/test_dashboard/mod.rs | 60 ++++------------------ src/ci/citool/templates/test_suites.askama | 17 ++++-- 2 files changed, 22 insertions(+), 55 deletions(-) diff --git a/src/ci/citool/src/test_dashboard/mod.rs b/src/ci/citool/src/test_dashboard/mod.rs index ad9fe029e15df..163e9c1acea0e 100644 --- a/src/ci/citool/src/test_dashboard/mod.rs +++ b/src/ci/citool/src/test_dashboard/mod.rs @@ -10,22 +10,6 @@ use crate::jobs::JobDatabase; use crate::metrics::{JobMetrics, JobName, download_auto_job_metrics, get_test_suites}; use crate::utils::normalize_path_delimiters; -pub struct TestInfo { - name: String, - jobs: Vec, -} - -struct JobTestResult { - job_name: String, - outcome: TestOutcome, -} - -#[derive(Default)] -struct TestSuiteInfo { - name: String, - tests: BTreeMap, -} - /// Generate a set of HTML files into a directory that contain a dashboard of test results. pub fn generate_test_dashboard( db: JobDatabase, @@ -33,7 +17,6 @@ pub fn generate_test_dashboard( output_dir: &Path, ) -> anyhow::Result<()> { let metrics = download_auto_job_metrics(&db, None, current)?; - let suites = gather_test_suites(&metrics); std::fs::create_dir_all(output_dir)?; @@ -52,27 +35,27 @@ fn write_page(dir: &Path, name: &str, template: &T) -> anyhow::Resu fn gather_test_suites(job_metrics: &HashMap) -> TestSuites { struct CoarseTestSuite<'a> { - kind: TestSuiteKind, tests: BTreeMap>, } let mut suites: HashMap = HashMap::new(); // First, gather tests from all jobs, stages and targets, and aggregate them per suite + // Only work with compiletest suites. for (job, metrics) in job_metrics { let test_suites = get_test_suites(&metrics.current); for suite in test_suites { - let (suite_name, stage, target, kind) = match &suite.metadata { - TestSuiteMetadata::CargoPackage { crates, stage, target, .. } => { - (crates.join(","), *stage, target, TestSuiteKind::Cargo) + let (suite_name, stage, target) = match &suite.metadata { + TestSuiteMetadata::CargoPackage { .. } => { + continue; } TestSuiteMetadata::Compiletest { suite, stage, target, .. } => { - (suite.clone(), *stage, target, TestSuiteKind::Compiletest) + (suite.clone(), *stage, target) } }; let suite_entry = suites .entry(suite_name.clone()) - .or_insert_with(|| CoarseTestSuite { kind, tests: Default::default() }); + .or_insert_with(|| CoarseTestSuite { tests: Default::default() }); let test_metadata = TestMetadata { job, stage, target }; for test in &suite.tests { @@ -98,29 +81,13 @@ fn gather_test_suites(job_metrics: &HashMap) -> TestSuites // Then, split the suites per directory let mut suites = suites.into_iter().collect::>(); - suites.sort_by(|a, b| a.1.kind.cmp(&b.1.kind).then_with(|| a.0.cmp(&b.0))); + suites.sort_by(|a, b| a.0.cmp(&b.0)); let mut target_suites = vec![]; for (suite_name, suite) in suites { - let suite = match suite.kind { - TestSuiteKind::Compiletest => TestSuite { - name: suite_name.clone(), - kind: TestSuiteKind::Compiletest, - group: build_test_group(&suite_name, suite.tests), - }, - TestSuiteKind::Cargo => { - let mut tests: Vec<_> = suite.tests.into_iter().collect(); - tests.sort_by(|a, b| a.0.cmp(&b.0)); - TestSuite { - name: format!("[cargo] {}", suite_name.clone()), - kind: TestSuiteKind::Cargo, - group: TestGroup { - name: suite_name, - root_tests: tests.into_iter().map(|t| t.1).collect(), - groups: vec![], - }, - } - } + let suite = TestSuite { + name: suite_name.clone(), + group: build_test_group(&suite_name, suite.tests), }; target_suites.push(suite); } @@ -187,7 +154,6 @@ impl<'a> TestSuites<'a> { #[derive(serde::Serialize)] struct TestSuite<'a> { name: String, - kind: TestSuiteKind, group: TestGroup<'a>, } @@ -225,12 +191,6 @@ impl<'a> TestGroup<'a> { } } -#[derive(PartialEq, Eq, PartialOrd, Ord, serde::Serialize)] -enum TestSuiteKind { - Compiletest, - Cargo, -} - #[derive(Template)] #[template(path = "test_suites.askama")] struct TestSuitesPage<'a> { diff --git a/src/ci/citool/templates/test_suites.askama b/src/ci/citool/templates/test_suites.askama index a8cedc65f2434..bb3d9e363911d 100644 --- a/src/ci/citool/templates/test_suites.askama +++ b/src/ci/citool/templates/test_suites.askama @@ -1,10 +1,15 @@ {% extends "layout.askama" %} {% block content %} -

    Rust CI Test Dashboard

    +

    Rust CI test dashboard

    - Total tests: {{ test_count }} +
    +
    Total tests: {{ test_count }}
    +
    + To find tests that haven't been executed anywhere, click on "Open all" and search for "(0 passed". +
    +
    @@ -13,9 +18,7 @@
      {% for suite in suites.suites %} - {% if suite.kind == TestSuiteKind::Compiletest %} - {{ suite.group|safe }} - {% endif %} + {{ suite.group|safe }} {% endfor %}
    @@ -33,6 +36,10 @@ h1 { justify-content: space-between; } +.test-count { + font-size: 1.2em; +} + .test-suites { background: white; border-radius: 8px; From 1a6e0d52e5b008cfd48f78285bb3655ecfd5d73e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jakub=20Ber=C3=A1nek?= Date: Thu, 17 Apr 2025 17:14:26 +0200 Subject: [PATCH 06/12] Render test revisions separately --- src/ci/citool/src/test_dashboard/mod.rs | 43 ++++++++++++++++++----- src/ci/citool/templates/test_group.askama | 13 ++++++- 2 files changed, 46 insertions(+), 10 deletions(-) diff --git a/src/ci/citool/src/test_dashboard/mod.rs b/src/ci/citool/src/test_dashboard/mod.rs index 163e9c1acea0e..c16385baa3ba4 100644 --- a/src/ci/citool/src/test_dashboard/mod.rs +++ b/src/ci/citool/src/test_dashboard/mod.rs @@ -60,19 +60,27 @@ fn gather_test_suites(job_metrics: &HashMap) -> TestSuites for test in &suite.tests { let test_name = normalize_test_name(&test.name, &suite_name); - let test_entry = suite_entry - .tests - .entry(test_name.clone()) - .or_insert_with(|| Test { name: test_name, passed: vec![], ignored: vec![] }); + let (test_name, variant_name) = match test_name.rsplit_once('#') { + Some((name, variant)) => (name.to_string(), variant.to_string()), + None => (test_name, "".to_string()), + }; + let test_entry = suite_entry.tests.entry(test_name.clone()).or_insert_with(|| { + Test { name: test_name.clone(), revisions: Default::default() } + }); + let variant_entry = test_entry + .revisions + .entry(variant_name) + .or_insert_with(|| TestResults { passed: vec![], ignored: vec![] }); + match test.outcome { TestOutcome::Passed => { - test_entry.passed.push(test_metadata); + variant_entry.passed.push(test_metadata); } TestOutcome::Ignored { ignore_reason: _ } => { - test_entry.ignored.push(test_metadata); + variant_entry.ignored.push(test_metadata); } TestOutcome::Failed => { - eprintln!("Warning: failed test"); + eprintln!("Warning: failed test {test_name}"); } } } @@ -158,12 +166,29 @@ struct TestSuite<'a> { } #[derive(Debug, serde::Serialize)] -struct Test<'a> { - name: String, +struct TestResults<'a> { passed: Vec>, ignored: Vec>, } +#[derive(Debug, serde::Serialize)] +struct Test<'a> { + name: String, + revisions: BTreeMap>, +} + +impl<'a> Test<'a> { + /// If this is a test without revisions, it will have a single entry in `revisions` with + /// an empty string as the revision name. + fn single_test(&self) -> Option<&TestResults<'a>> { + if self.revisions.len() == 1 { + self.revisions.iter().next().take_if(|e| e.0.is_empty()).map(|e| e.1) + } else { + None + } + } +} + #[derive(Clone, Copy, Debug, serde::Serialize)] struct TestMetadata<'a> { job: &'a str, diff --git a/src/ci/citool/templates/test_group.askama b/src/ci/citool/templates/test_group.askama index a0b7fa863e577..535d98e0c247f 100644 --- a/src/ci/citool/templates/test_group.askama +++ b/src/ci/citool/templates/test_group.askama @@ -13,7 +13,18 @@ {% if !root_tests.is_empty() %}
      {% for test in root_tests %} -
    • {{ test.name }} ({{ test.passed.len() }} passed, {{ test.ignored.len() }} ignored)
      • +
    • + {% if let Some(result) = test.single_test() %} + {{ test.name }} ({{ result.passed.len() }} passed, {{ result.ignored.len() }} ignored) + {% else %} + {{ test.name }} ({{ test.revisions.len() }} revision{{ test.revisions.len() | pluralize }}) +
        + {% for (revision, result) in test.revisions %} +
      • #{{ revision }} ({{ result.passed.len() }} passed, {{ result.ignored.len() }} ignored)
        • + {% endfor %} +
      + {% endif %} +
      • {% endfor %}
    {% endif %} From d2c1763336080030a235dae56f6096e9deb2ec9f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jakub=20Ber=C3=A1nek?= Date: Thu, 17 Apr 2025 17:18:38 +0200 Subject: [PATCH 07/12] Create a macro for rendering test results --- src/ci/citool/templates/test_group.askama | 8 ++++++-- src/ci/citool/templates/test_suites.askama | 2 +- 2 files changed, 7 insertions(+), 3 deletions(-) diff --git a/src/ci/citool/templates/test_group.askama b/src/ci/citool/templates/test_group.askama index 535d98e0c247f..ba19d9258d8a0 100644 --- a/src/ci/citool/templates/test_group.askama +++ b/src/ci/citool/templates/test_group.askama @@ -1,3 +1,7 @@ +{% macro test_result(r) -%} +passed: {{ r.passed.len() }}, ignored: {{ r.ignored.len() }} +{%- endmacro %} +
  • {{ name }} ({{ test_count() }} test{{ test_count() | pluralize }}) @@ -15,12 +19,12 @@ {% for test in root_tests %}
  • {% if let Some(result) = test.single_test() %} - {{ test.name }} ({{ result.passed.len() }} passed, {{ result.ignored.len() }} ignored) + {{ test.name }} ({% call test_result(result) %}) {% else %} {{ test.name }} ({{ test.revisions.len() }} revision{{ test.revisions.len() | pluralize }})
      {% for (revision, result) in test.revisions %} -
    • #{{ revision }} ({{ result.passed.len() }} passed, {{ result.ignored.len() }} ignored)
      • +
    • #{{ revision }} ({% call test_result(result) %})
      • {% endfor %}
    {% endif %} diff --git a/src/ci/citool/templates/test_suites.askama b/src/ci/citool/templates/test_suites.askama index bb3d9e363911d..d36e85228e2b0 100644 --- a/src/ci/citool/templates/test_suites.askama +++ b/src/ci/citool/templates/test_suites.askama @@ -7,7 +7,7 @@
    Total tests: {{ test_count }}
    - To find tests that haven't been executed anywhere, click on "Open all" and search for "(0 passed". + To find tests that haven't been executed anywhere, click on "Open all" and search for "passed: 0".
    From aa9cb70190bb38e466983abf0c53c6f26afab4de Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jakub=20Ber=C3=A1nek?= Date: Thu, 17 Apr 2025 17:25:12 +0200 Subject: [PATCH 08/12] Print number of root tests and subdirectories --- src/ci/citool/templates/test_group.askama | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/src/ci/citool/templates/test_group.askama b/src/ci/citool/templates/test_group.askama index ba19d9258d8a0..bdf32d00f4a19 100644 --- a/src/ci/citool/templates/test_group.askama +++ b/src/ci/citool/templates/test_group.askama @@ -4,7 +4,12 @@ passed: {{ r.passed.len() }}, ignored: {{ r.ignored.len() }}
  • -{{ name }} ({{ test_count() }} test{{ test_count() | pluralize }}) +{{ name }} ({{ test_count() }} test{{ test_count() | pluralize }}{% if !root_tests.is_empty() && root_tests.len() as u64 != test_count() -%} + , {{ root_tests.len() }} root test{{ root_tests.len() | pluralize }} +{%- endif %}{% if !groups.is_empty() -%} + , {{ groups.len() }} subdir{{ groups.len() | pluralize }} +{%- endif %}) + {% if !groups.is_empty() %}
      From 08cb187d263ec91e8e6d35b4b235eee4ecad3357 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jakub=20Ber=C3=A1nek?= Date: Thu, 17 Apr 2025 17:26:40 +0200 Subject: [PATCH 09/12] Turn `test_dashboard` into a file --- src/ci/citool/src/{test_dashboard/mod.rs => test_dashboard.rs} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename src/ci/citool/src/{test_dashboard/mod.rs => test_dashboard.rs} (100%) diff --git a/src/ci/citool/src/test_dashboard/mod.rs b/src/ci/citool/src/test_dashboard.rs similarity index 100% rename from src/ci/citool/src/test_dashboard/mod.rs rename to src/ci/citool/src/test_dashboard.rs From cecf16785f193c247c47e42524322aeccc96c8a4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jakub=20Ber=C3=A1nek?= Date: Thu, 17 Apr 2025 17:38:15 +0200 Subject: [PATCH 10/12] Add a note about the test dashboard to the post-merge report --- src/ci/citool/src/main.rs | 16 +++++++++++++++- 1 file changed, 15 insertions(+), 1 deletion(-) diff --git a/src/ci/citool/src/main.rs b/src/ci/citool/src/main.rs index a7a289fc3d4b1..f4e671b609fa6 100644 --- a/src/ci/citool/src/main.rs +++ b/src/ci/citool/src/main.rs @@ -24,7 +24,7 @@ use crate::github::JobInfoResolver; use crate::jobs::RunType; use crate::metrics::{JobMetrics, download_auto_job_metrics, download_job_metrics, load_metrics}; use crate::test_dashboard::generate_test_dashboard; -use crate::utils::load_env_var; +use crate::utils::{load_env_var, output_details}; const CI_DIRECTORY: &str = concat!(env!("CARGO_MANIFEST_DIR"), "/.."); const DOCKER_DIRECTORY: &str = concat!(env!("CARGO_MANIFEST_DIR"), "/../docker"); @@ -188,6 +188,20 @@ fn post_merge_report(db: JobDatabase, current: String, parent: String) -> anyhow let mut job_info_resolver = JobInfoResolver::new(); output_test_diffs(&metrics, &mut job_info_resolver); + + output_details("Test dashboard", || { + println!( + r#"\nRun + +```bash +cargo run --manifest-path src/ci/citool/Cargo.toml -- \ + test-dashboard {current} --output-dir test-dashboard +``` +And then open `test-dashboard/index.html` in your browser to see an overview of all executed tests. +"# + ); + }); + output_largest_duration_changes(&metrics, &mut job_info_resolver); Ok(()) From 65ce38a4c96da2f950fb6b181b2b83b0320d103d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jakub=20Ber=C3=A1nek?= Date: Fri, 18 Apr 2025 12:32:19 +0200 Subject: [PATCH 11/12] Add a note that explains the counts --- src/ci/citool/templates/test_suites.askama | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/src/ci/citool/templates/test_suites.askama b/src/ci/citool/templates/test_suites.askama index d36e85228e2b0..4997f6a3f1c9a 100644 --- a/src/ci/citool/templates/test_suites.askama +++ b/src/ci/citool/templates/test_suites.askama @@ -2,6 +2,10 @@ {% block content %}

      Rust CI test dashboard

      +
      +Here's how to interpret the "passed" and "ignored" counts: +the count includes all combinations of "stage" x "target" x "CI job where the test was executed or ignored". +
      From b18e37305304780530224736ad55145063c7e8a6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jakub=20Ber=C3=A1nek?= Date: Fri, 18 Apr 2025 12:44:39 +0200 Subject: [PATCH 12/12] Reduce duplicated test prefixes in nested subdirectories `assembly/asm` contained a test named `asm/aarch64-el2vmsa.rs`, while it should have been only `arch64-el2vmsa.rs`. --- src/ci/citool/src/test_dashboard.rs | 36 +++++++++-------------- src/ci/citool/templates/test_group.askama | 6 ++-- 2 files changed, 17 insertions(+), 25 deletions(-) diff --git a/src/ci/citool/src/test_dashboard.rs b/src/ci/citool/src/test_dashboard.rs index c16385baa3ba4..8fbd0d3f200d4 100644 --- a/src/ci/citool/src/test_dashboard.rs +++ b/src/ci/citool/src/test_dashboard.rs @@ -64,9 +64,10 @@ fn gather_test_suites(job_metrics: &HashMap) -> TestSuites Some((name, variant)) => (name.to_string(), variant.to_string()), None => (test_name, "".to_string()), }; - let test_entry = suite_entry.tests.entry(test_name.clone()).or_insert_with(|| { - Test { name: test_name.clone(), revisions: Default::default() } - }); + let test_entry = suite_entry + .tests + .entry(test_name.clone()) + .or_insert_with(|| Test { revisions: Default::default() }); let variant_entry = test_entry .revisions .entry(variant_name) @@ -91,16 +92,12 @@ fn gather_test_suites(job_metrics: &HashMap) -> TestSuites let mut suites = suites.into_iter().collect::>(); suites.sort_by(|a, b| a.0.cmp(&b.0)); - let mut target_suites = vec![]; - for (suite_name, suite) in suites { - let suite = TestSuite { - name: suite_name.clone(), - group: build_test_group(&suite_name, suite.tests), - }; - target_suites.push(suite); - } + let suites = suites + .into_iter() + .map(|(suite_name, suite)| TestSuite { group: build_test_group(&suite_name, suite.tests) }) + .collect(); - TestSuites { suites: target_suites } + TestSuites { suites } } /// Recursively expand a test group based on filesystem hierarchy. @@ -115,7 +112,7 @@ fn build_test_group<'a>(name: &str, tests: BTreeMap>) -> TestGr if components.peek().is_none() { // This is a root test - root_tests.push(test); + root_tests.push((name, test)); } else { // This is a test in a nested directory let subdir_tests = @@ -148,7 +145,6 @@ fn normalize_test_name(name: &str, suite_name: &str) -> String { name.trim_start_matches("/").to_string() } -#[derive(serde::Serialize)] struct TestSuites<'a> { suites: Vec>, } @@ -159,21 +155,16 @@ impl<'a> TestSuites<'a> { } } -#[derive(serde::Serialize)] struct TestSuite<'a> { - name: String, group: TestGroup<'a>, } -#[derive(Debug, serde::Serialize)] struct TestResults<'a> { passed: Vec>, ignored: Vec>, } -#[derive(Debug, serde::Serialize)] struct Test<'a> { - name: String, revisions: BTreeMap>, } @@ -189,7 +180,8 @@ impl<'a> Test<'a> { } } -#[derive(Clone, Copy, Debug, serde::Serialize)] +#[derive(Clone, Copy)] +#[allow(dead_code)] struct TestMetadata<'a> { job: &'a str, stage: u32, @@ -198,13 +190,13 @@ struct TestMetadata<'a> { // We have to use a template for the TestGroup instead of a macro, because // macros cannot be recursive in askama at the moment. -#[derive(Template, serde::Serialize)] +#[derive(Template)] #[template(path = "test_group.askama")] /// Represents a group of tests struct TestGroup<'a> { name: String, /// Tests located directly in this directory - root_tests: Vec>, + root_tests: Vec<(String, Test<'a>)>, /// Nested directories with additional tests groups: Vec<(String, TestGroup<'a>)>, } diff --git a/src/ci/citool/templates/test_group.askama b/src/ci/citool/templates/test_group.askama index bdf32d00f4a19..95731103f3b9d 100644 --- a/src/ci/citool/templates/test_group.askama +++ b/src/ci/citool/templates/test_group.askama @@ -21,12 +21,12 @@ passed: {{ r.passed.len() }}, ignored: {{ r.ignored.len() }} {% if !root_tests.is_empty() %}
        - {% for test in root_tests %} + {% for (name, test) in root_tests %}
      • {% if let Some(result) = test.single_test() %} - {{ test.name }} ({% call test_result(result) %}) + {{ name }} ({% call test_result(result) %}) {% else %} - {{ test.name }} ({{ test.revisions.len() }} revision{{ test.revisions.len() | pluralize }}) + {{ name }} ({{ test.revisions.len() }} revision{{ test.revisions.len() | pluralize }})
          {% for (revision, result) in test.revisions %}
        • #{{ revision }} ({% call test_result(result) %})