Skip to content

Commit 6463590

Browse files
committed
Postprocess test suite metrics into GitHub summary
1 parent fd17dea commit 6463590

File tree

5 files changed

+181
-0
lines changed

5 files changed

+181
-0
lines changed

Diff for: .github/workflows/ci.yml

+17
Original file line numberDiff line numberDiff line change
@@ -182,6 +182,13 @@ jobs:
182182
- name: show the current environment
183183
run: src/ci/scripts/dump-environment.sh
184184

185+
# Pre-build citool before the following step uninstalls rustup
186+
# Build is into the build directory, to avoid modifying sources
187+
- name: build citool
188+
run: |
189+
cd src/ci/citool
190+
CARGO_TARGET_DIR=../../../build/citool cargo build
191+
185192
- name: run the build
186193
# Redirect stderr to stdout to avoid reordering the two streams in the GHA logs.
187194
run: src/ci/scripts/run-build-from-ci.sh 2>&1
@@ -218,6 +225,16 @@ jobs:
218225
# erroring about invalid credentials instead.
219226
if: github.event_name == 'push' || env.DEPLOY == '1' || env.DEPLOY_ALT == '1'
220227

228+
- name: postprocess metrics into the summary
229+
run: |
230+
if [ -f build/metrics.json ]; then
231+
./build/citool/debug/citool postprocess-metrics build/metrics.json ${GITHUB_STEP_SUMMARY}
232+
elif [ -f obj/build/metrics.json ]; then
233+
./build/citool/debug/citool postprocess-metrics obj/build/metrics.json ${GITHUB_STEP_SUMMARY}
234+
else
235+
echo "No metrics.json found"
236+
fi
237+
221238
- name: upload job metrics to DataDog
222239
if: needs.calculate_matrix.outputs.run_type != 'pr'
223240
env:

Diff for: src/ci/citool/Cargo.lock

+9
Original file line numberDiff line numberDiff line change
@@ -58,11 +58,20 @@ version = "1.0.95"
5858
source = "registry+https://github.com/rust-lang/crates.io-index"
5959
checksum = "34ac096ce696dc2fcabef30516bb13c0a68a11d30131d3df6f04711467681b04"
6060

61+
[[package]]
62+
name = "build_helper"
63+
version = "0.1.0"
64+
dependencies = [
65+
"serde",
66+
"serde_derive",
67+
]
68+
6169
[[package]]
6270
name = "citool"
6371
version = "0.1.0"
6472
dependencies = [
6573
"anyhow",
74+
"build_helper",
6675
"clap",
6776
"insta",
6877
"serde",

Diff for: src/ci/citool/Cargo.toml

+2
Original file line numberDiff line numberDiff line change
@@ -10,6 +10,8 @@ serde = { version = "1", features = ["derive"] }
1010
serde_yaml = "0.9"
1111
serde_json = "1"
1212

13+
build_helper = { path = "../../build_helper" }
14+
1315
[dev-dependencies]
1416
insta = "1"
1517

Diff for: src/ci/citool/src/main.rs

+15
Original file line numberDiff line numberDiff line change
@@ -1,3 +1,5 @@
1+
mod metrics;
2+
13
use std::collections::BTreeMap;
24
use std::path::{Path, PathBuf};
35
use std::process::Command;
@@ -6,6 +8,8 @@ use anyhow::Context;
68
use clap::Parser;
79
use serde_yaml::Value;
810

11+
use crate::metrics::postprocess_metrics;
12+
913
const CI_DIRECTORY: &str = concat!(env!("CARGO_MANIFEST_DIR"), "/..");
1014
const DOCKER_DIRECTORY: &str = concat!(env!("CARGO_MANIFEST_DIR"), "/../docker");
1115
const JOBS_YML_PATH: &str = concat!(env!("CARGO_MANIFEST_DIR"), "/../github-actions/jobs.yml");
@@ -338,6 +342,14 @@ enum Args {
338342
#[clap(long = "type", default_value = "auto")]
339343
job_type: JobType,
340344
},
345+
/// Postprocess the metrics.json file generated by bootstrap.
346+
PostprocessMetrics {
347+
/// Path to the metrics.json file
348+
metrics_path: PathBuf,
349+
/// Path to a file where the postprocessed metrics summary will be stored.
350+
/// Usually, this will be GITHUB_STEP_SUMMARY on CI.
351+
summary_path: PathBuf,
352+
},
341353
}
342354

343355
#[derive(clap::ValueEnum, Clone)]
@@ -369,6 +381,9 @@ fn main() -> anyhow::Result<()> {
369381
Args::RunJobLocally { job_type, name } => {
370382
run_workflow_locally(load_db(default_jobs_file)?, job_type, name)?
371383
}
384+
Args::PostprocessMetrics { metrics_path, summary_path } => {
385+
postprocess_metrics(&metrics_path, &summary_path)?;
386+
}
372387
}
373388

374389
Ok(())

Diff for: src/ci/citool/src/metrics.rs

+138
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,138 @@
1+
use std::collections::BTreeMap;
2+
use std::fs::File;
3+
use std::io::Write;
4+
use std::path::Path;
5+
6+
use anyhow::Context;
7+
use build_helper::metrics::{JsonNode, JsonRoot, TestOutcome, TestSuite, TestSuiteMetadata};
8+
9+
pub fn postprocess_metrics(metrics_path: &Path, summary_path: &Path) -> anyhow::Result<()> {
10+
let metrics = load_metrics(metrics_path)?;
11+
12+
let mut file = File::options()
13+
.append(true)
14+
.create(true)
15+
.open(summary_path)
16+
.with_context(|| format!("Cannot open summary file at {summary_path:?}"))?;
17+
18+
record_test_suites(&metrics, &mut file)?;
19+
20+
Ok(())
21+
}
22+
23+
fn record_test_suites(metrics: &JsonRoot, file: &mut File) -> anyhow::Result<()> {
24+
let suites = get_test_suites(&metrics);
25+
26+
if !suites.is_empty() {
27+
let aggregated = aggregate_test_suites(&suites);
28+
let table = render_table(aggregated);
29+
writeln!(file, "\n# Test results\n")?;
30+
writeln!(file, "{table}")?;
31+
} else {
32+
eprintln!("No test suites found in metrics");
33+
}
34+
35+
Ok(())
36+
}
37+
38+
fn render_table(suites: BTreeMap<String, TestSuiteRecord>) -> String {
39+
use std::fmt::Write;
40+
41+
let mut table = "| Test suite | Passed ✅ | Ignored 🚫 | Failed ❌ |\n".to_string();
42+
writeln!(table, "|:------|------:|------:|------:|").unwrap();
43+
44+
fn write_row(
45+
buffer: &mut String,
46+
name: &str,
47+
record: &TestSuiteRecord,
48+
surround: &str,
49+
) -> std::fmt::Result {
50+
let TestSuiteRecord { passed, ignored, failed } = record;
51+
let total = (record.passed + record.ignored + record.failed) as f64;
52+
let passed_pct = ((*passed as f64) / total) * 100.0;
53+
let ignored_pct = ((*ignored as f64) / total) * 100.0;
54+
let failed_pct = ((*failed as f64) / total) * 100.0;
55+
56+
write!(buffer, "| {surround}{name}{surround} |")?;
57+
write!(buffer, " {surround}{passed} ({passed_pct:.0}%){surround} |")?;
58+
write!(buffer, " {surround}{ignored} ({ignored_pct:.0}%){surround} |")?;
59+
writeln!(buffer, " {surround}{failed} ({failed_pct:.0}%){surround} |")?;
60+
61+
Ok(())
62+
}
63+
64+
let mut total = TestSuiteRecord::default();
65+
for (name, record) in suites {
66+
write_row(&mut table, &name, &record, "").unwrap();
67+
total.passed += record.passed;
68+
total.ignored += record.ignored;
69+
total.failed += record.failed;
70+
}
71+
write_row(&mut table, "Total", &total, "**").unwrap();
72+
table
73+
}
74+
75+
#[derive(Default)]
76+
struct TestSuiteRecord {
77+
passed: u64,
78+
ignored: u64,
79+
failed: u64,
80+
}
81+
82+
fn aggregate_test_suites(suites: &[&TestSuite]) -> BTreeMap<String, TestSuiteRecord> {
83+
let mut records: BTreeMap<String, TestSuiteRecord> = BTreeMap::new();
84+
for suite in suites {
85+
let name = match &suite.metadata {
86+
TestSuiteMetadata::CargoPackage { crates, stage, .. } => {
87+
format!("{} (stage {stage})", crates.join(", "))
88+
}
89+
TestSuiteMetadata::Compiletest { suite, stage, .. } => {
90+
format!("{suite} (stage {stage})")
91+
}
92+
};
93+
let record = records.entry(name).or_default();
94+
for test in &suite.tests {
95+
match test.outcome {
96+
TestOutcome::Passed => {
97+
record.passed += 1;
98+
}
99+
TestOutcome::Failed => {
100+
record.failed += 1;
101+
}
102+
TestOutcome::Ignored { .. } => {
103+
record.ignored += 1;
104+
}
105+
}
106+
}
107+
}
108+
records
109+
}
110+
111+
fn get_test_suites(metrics: &JsonRoot) -> Vec<&TestSuite> {
112+
fn visit_test_suites<'a>(nodes: &'a [JsonNode], suites: &mut Vec<&'a TestSuite>) {
113+
for node in nodes {
114+
match node {
115+
JsonNode::RustbuildStep { children, .. } => {
116+
visit_test_suites(&children, suites);
117+
}
118+
JsonNode::TestSuite(suite) => {
119+
suites.push(&suite);
120+
}
121+
}
122+
}
123+
}
124+
125+
let mut suites = vec![];
126+
for invocation in &metrics.invocations {
127+
visit_test_suites(&invocation.children, &mut suites);
128+
}
129+
suites
130+
}
131+
132+
fn load_metrics(path: &Path) -> anyhow::Result<JsonRoot> {
133+
let metrics = std::fs::read_to_string(path)
134+
.with_context(|| format!("Cannot read JSON metrics from {path:?}"))?;
135+
let metrics: JsonRoot = serde_json::from_str(&metrics)
136+
.with_context(|| format!("Cannot deserialize JSON metrics from {path:?}"))?;
137+
Ok(metrics)
138+
}

0 commit comments

Comments
 (0)