Add citool command for generating a test dashboard

src/ci/citool/Cargo.lock

@@ -64,12 +64,63 @@ version = "1.0.95"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "34ac096ce696dc2fcabef30516bb13c0a68a11d30131d3df6f04711467681b04"
 
+[[package]]
+name = "askama"
+version = "0.13.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "5d4744ed2eef2645831b441d8f5459689ade2ab27c854488fbab1fbe94fce1a7"
+dependencies = [
+ "askama_derive",
+ "itoa",
+ "percent-encoding",
+ "serde",
+ "serde_json",
+]
+
+[[package]]
+name = "askama_derive"
+version = "0.13.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "d661e0f57be36a5c14c48f78d09011e67e0cb618f269cca9f2fd8d15b68c46ac"
+dependencies = [
+ "askama_parser",
+ "basic-toml",
+ "memchr",
+ "proc-macro2",
+ "quote",
+ "rustc-hash",
+ "serde",
+ "serde_derive",
+ "syn",
+]
+
+[[package]]
+name = "askama_parser"
+version = "0.13.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "cf315ce6524c857bb129ff794935cf6d42c82a6cff60526fe2a63593de4d0d4f"
+dependencies = [
+ "memchr",
+ "serde",
+ "serde_derive",
+ "winnow",
+]
+
 [[package]]
 name = "base64"
 version = "0.22.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "72b3254f16251a8381aa12e40e3c4d2f0199f8c6508fbecb9d91f575e0fbb8c6"
 
+[[package]]
+name = "basic-toml"
+version = "0.1.10"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "ba62675e8242a4c4e806d12f11d136e626e6c8361d6b829310732241652a178a"
+dependencies = [
+ "serde",
+]
+
 [[package]]
 name = "build_helper"
 version = "0.1.0"
@@ -104,6 +155,7 @@ name = "citool"
 version = "0.1.0"
 dependencies = [
  "anyhow",
+ "askama",
  "build_helper",
  "clap",
  "csv",
@@ -646,6 +698,12 @@ dependencies = [
  "windows-sys 0.52.0",
 ]
 
+[[package]]
+name = "rustc-hash"
+version = "2.1.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "357703d41365b4b27c590e3ed91eabb1b663f07c4c084095e60cbed4362dff0d"
+
 [[package]]
 name = "rustls"
 version = "0.23.23"
@@ -1026,6 +1084,15 @@ version = "0.52.6"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "589f6da84c646204747d1270a2a5661ea66ed1cced2631d546fdfb155959f9ec"
 
+[[package]]
+name = "winnow"
+version = "0.7.6"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "63d3fcd9bba44b03821e7d699eeee959f3126dcc4aa8e4ae18ec617c2a5cea10"
+dependencies = [
+ "memchr",
+]
+
 [[package]]
 name = "write16"
 version = "1.0.0"

src/ci/citool/Cargo.toml

@@ -5,6 +5,7 @@ edition = "2021"
 
 [dependencies]
 anyhow = "1"
+askama = "0.13"
 clap = { version = "4.5", features = ["derive"] }
 csv = "1"
 diff = "0.1"

src/ci/citool/src/main.rs

@@ -4,6 +4,7 @@ mod datadog;
 mod github;
 mod jobs;
 mod metrics;
+mod test_dashboard;
 mod utils;
 
 use std::collections::{BTreeMap, HashMap};
@@ -22,6 +23,7 @@ use crate::datadog::upload_datadog_metric;
 use crate::github::JobInfoResolver;
 use crate::jobs::RunType;
 use crate::metrics::{JobMetrics, download_auto_job_metrics, download_job_metrics, load_metrics};
+use crate::test_dashboard::generate_test_dashboard;
 use crate::utils::load_env_var;
 
 const CI_DIRECTORY: &str = concat!(env!("CARGO_MANIFEST_DIR"), "/..");
@@ -234,6 +236,14 @@ enum Args {
         /// Current commit that will be compared to `parent`.
         current: String,
     },
+    /// Generate a directory containing a HTML dashboard of test results from a CI run.
+    TestDashboard {
+        /// Commit SHA that was tested on CI to analyze.
+        current: String,
+        /// Output path for the HTML directory.
+        #[clap(long)]
+        output_dir: PathBuf,
+    },
 }
 
 #[derive(clap::ValueEnum, Clone)]
@@ -275,7 +285,11 @@ fn main() -> anyhow::Result<()> {
             postprocess_metrics(metrics_path, parent, job_name)?;
         }
         Args::PostMergeReport { current, parent } => {
-            post_merge_report(load_db(default_jobs_file)?, current, parent)?;
+            post_merge_report(load_db(&default_jobs_file)?, current, parent)?;
+        }
+        Args::TestDashboard { current, output_dir } => {
+            let db = load_db(&default_jobs_file)?;
+            generate_test_dashboard(db, &current, &output_dir)?;
         }
     }
 

src/ci/citool/src/test_dashboard/mod.rs

@@ -0,0 +1,239 @@
+use std::collections::{BTreeMap, HashMap};
+use std::fs::File;
+use std::io::BufWriter;
+use std::path::{Path, PathBuf};
+
+use askama::Template;
+use build_helper::metrics::{TestOutcome, TestSuiteMetadata};
+
+use crate::jobs::JobDatabase;
+use crate::metrics::{JobMetrics, JobName, download_auto_job_metrics, get_test_suites};
+use crate::utils::normalize_path_delimiters;
+
+pub struct TestInfo {
+    name: String,
+    jobs: Vec<JobTestResult>,
+}
+
+struct JobTestResult {
+    job_name: String,
+    outcome: TestOutcome,
+}
+
+#[derive(Default)]
+struct TestSuiteInfo {
+    name: String,
+    tests: BTreeMap<String, TestInfo>,
+}
+
+/// Generate a set of HTML files into a directory that contain a dashboard of test results.
+pub fn generate_test_dashboard(
+    db: JobDatabase,
+    current: &str,
+    output_dir: &Path,
+) -> anyhow::Result<()> {
+    let metrics = download_auto_job_metrics(&db, None, current)?;
+
+    let suites = gather_test_suites(&metrics);
+
+    std::fs::create_dir_all(output_dir)?;
+
+    let test_count = suites.test_count();
+    write_page(output_dir, "index.html", &TestSuitesPage { suites, test_count })?;
+
+    Ok(())
+}
+
+fn write_page<T: Template>(dir: &Path, name: &str, template: &T) -> anyhow::Result<()> {
+    let mut file = BufWriter::new(File::create(dir.join(name))?);
+    Template::write_into(template, &mut file)?;
+    Ok(())
+}
+
+fn gather_test_suites(job_metrics: &HashMap<JobName, JobMetrics>) -> TestSuites {
+    struct CoarseTestSuite<'a> {
+        kind: TestSuiteKind,
+        tests: BTreeMap<String, Test<'a>>,
+    }
+
+    let mut suites: HashMap<String, CoarseTestSuite> = HashMap::new();
+
+    // First, gather tests from all jobs, stages and targets, and aggregate them per suite
+    for (job, metrics) in job_metrics {
+        let test_suites = get_test_suites(&metrics.current);
+        for suite in test_suites {
+            let (suite_name, stage, target, kind) = match &suite.metadata {
+                TestSuiteMetadata::CargoPackage { crates, stage, target, .. } => {
+                    (crates.join(","), *stage, target, TestSuiteKind::Cargo)
+                }
+                TestSuiteMetadata::Compiletest { suite, stage, target, .. } => {
+                    (suite.clone(), *stage, target, TestSuiteKind::Compiletest)
+                }
+            };
+            let suite_entry = suites
+                .entry(suite_name.clone())
+                .or_insert_with(|| CoarseTestSuite { kind, tests: Default::default() });
+            let test_metadata = TestMetadata { job, stage, target };
+
+            for test in &suite.tests {
+                let test_name = normalize_test_name(&test.name, &suite_name);
+                let test_entry = suite_entry
+                    .tests
+                    .entry(test_name.clone())
+                    .or_insert_with(|| Test { name: test_name, passed: vec![], ignored: vec![] });
+                match test.outcome {
+                    TestOutcome::Passed => {
+                        test_entry.passed.push(test_metadata);
+                    }
+                    TestOutcome::Ignored { ignore_reason: _ } => {
+                        test_entry.ignored.push(test_metadata);
+                    }
+                    TestOutcome::Failed => {
+                        eprintln!("Warning: failed test");
+                    }
+                }
+            }
+        }
+    }
+
+    // Then, split the suites per directory
+    let mut suites = suites.into_iter().collect::<Vec<_>>();
+    suites.sort_by(|a, b| a.1.kind.cmp(&b.1.kind).then_with(|| a.0.cmp(&b.0)));
+
+    let mut target_suites = vec![];
+    for (suite_name, suite) in suites {
+        let suite = match suite.kind {
+            TestSuiteKind::Compiletest => TestSuite {
+                name: suite_name.clone(),
+                kind: TestSuiteKind::Compiletest,
+                group: build_test_group(&suite_name, suite.tests),
+            },
+            TestSuiteKind::Cargo => {
+                let mut tests: Vec<_> = suite.tests.into_iter().collect();
+                tests.sort_by(|a, b| a.0.cmp(&b.0));
+                TestSuite {
+                    name: format!("[cargo] {}", suite_name.clone()),
+                    kind: TestSuiteKind::Cargo,
+                    group: TestGroup {
+                        name: suite_name,
+                        root_tests: tests.into_iter().map(|t| t.1).collect(),
+                        groups: vec![],
+                    },
+                }
+            }
+        };
+        target_suites.push(suite);
+    }
+
+    TestSuites { suites: target_suites }
+}
+
+/// Recursively expand a test group based on filesystem hierarchy.
+fn build_test_group<'a>(name: &str, tests: BTreeMap<String, Test<'a>>) -> TestGroup<'a> {
+    let mut root_tests = vec![];
+    let mut subdirs: BTreeMap<String, BTreeMap<String, Test<'a>>> = Default::default();
+
+    // Split tests into root tests and tests located in subdirectories
+    for (name, test) in tests {
+        let mut components = Path::new(&name).components().peekable();
+        let subdir = components.next().unwrap();
+
+        if components.peek().is_none() {
+            // This is a root test
+            root_tests.push(test);
+        } else {
+            // This is a test in a nested directory
+            let subdir_tests =
+                subdirs.entry(subdir.as_os_str().to_str().unwrap().to_string()).or_default();
+            let test_name =
+                components.into_iter().collect::<PathBuf>().to_str().unwrap().to_string();
+            subdir_tests.insert(test_name, test);
+        }
+    }
+    let dirs = subdirs
+        .into_iter()
+        .map(|(name, tests)| {
+            let group = build_test_group(&name, tests);
+            (name, group)
+        })
+        .collect();
+
+    TestGroup { name: name.to_string(), root_tests, groups: dirs }
+}
+
+/// Compiletest tests start with `[suite] tests/[suite]/a/b/c...`.
+/// Remove the `[suite] tests/[suite]/` prefix so that we can find the filesystem path.
+/// Also normalizes path delimiters.
+fn normalize_test_name(name: &str, suite_name: &str) -> String {
+    let name = normalize_path_delimiters(name);
+    let name = name.as_ref();
+    let name = name.strip_prefix(&format!("[{suite_name}]")).unwrap_or(name).trim();
+    let name = name.strip_prefix("tests/").unwrap_or(name);
+    let name = name.strip_prefix(suite_name).unwrap_or(name);
+    name.trim_start_matches("/").to_string()
+}
+
+#[derive(serde::Serialize)]
+struct TestSuites<'a> {
+    suites: Vec<TestSuite<'a>>,
+}
+
+impl<'a> TestSuites<'a> {
+    fn test_count(&self) -> u64 {
+        self.suites.iter().map(|suite| suite.group.test_count()).sum::<u64>()
+    }
+}
+
+#[derive(serde::Serialize)]
+struct TestSuite<'a> {
+    name: String,
+    kind: TestSuiteKind,
+    group: TestGroup<'a>,
+}
+
+#[derive(Debug, serde::Serialize)]
+struct Test<'a> {
+    name: String,
+    passed: Vec<TestMetadata<'a>>,
+    ignored: Vec<TestMetadata<'a>>,
+}
+
+#[derive(Clone, Copy, Debug, serde::Serialize)]
+struct TestMetadata<'a> {
+    job: &'a str,
+    stage: u32,
+    target: &'a str,
+}
+
+// We have to use a template for the TestGroup instead of a macro, because
+// macros cannot be recursive in askama at the moment.
+#[derive(Template, serde::Serialize)]
+#[template(path = "test_group.askama")]
+/// Represents a group of tests
+struct TestGroup<'a> {
+    name: String,
+    /// Tests located directly in this directory
+    root_tests: Vec<Test<'a>>,
+    /// Nested directories with additional tests
+    groups: Vec<(String, TestGroup<'a>)>,
+}
+
+impl<'a> TestGroup<'a> {
+    fn test_count(&self) -> u64 {
+        let root = self.root_tests.len() as u64;
+        self.groups.iter().map(|(_, group)| group.test_count()).sum::<u64>() + root
+    }
+}
+
+#[derive(PartialEq, Eq, PartialOrd, Ord, serde::Serialize)]
+enum TestSuiteKind {
+    Compiletest,
+    Cargo,
+}
+
+#[derive(Template)]
+#[template(path = "test_suites.askama")]
+struct TestSuitesPage<'a> {
+    suites: TestSuites<'a>,
+    test_count: u64,
+}