Skip to content

Commit f919e45

Browse files
authored
Generate Kani Metrics (#235)
Add a new "Kani Metrics" workflow (that runs every week) that calls `./scripts/run-kani --run metrics`, then creates a pull request to this repository with the computed metrics. See [here](https://github.com/carolynzech/verify-rust-std/pull/38) for an example of what the pull request will look like. Also update our "Build Book" workflow to publish graphs of the metrics. Callouts: - This is a separate workflow from the Kani workflow because it is a cronjob instead of running on every pull request. I thought the plots would be too noisy if we ran on every PR. - See the "Notes" section of `kani_std_analysis.py` for other callouts. By submitting this pull request, I confirm that my contribution is made under the terms of the Apache 2.0 and MIT licenses.
1 parent 2b2baa8 commit f919e45

File tree

10 files changed

+518
-5
lines changed

10 files changed

+518
-5
lines changed

.github/workflows/kani-metrics.yml

+42
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,42 @@
1+
name: Kani Metrics Update
2+
3+
on:
4+
schedule:
5+
- cron: '0 0 * * 0' # Run at 00:00 UTC every Sunday
6+
workflow_dispatch:
7+
8+
defaults:
9+
run:
10+
shell: bash
11+
12+
jobs:
13+
update-kani-metrics:
14+
runs-on: ubuntu-latest
15+
16+
steps:
17+
- name: Checkout Repository
18+
uses: actions/checkout@v4
19+
with:
20+
submodules: true
21+
22+
# The Kani metrics collection uses a Python script (kani_std_analysis.py), so make sure Python is installed
23+
- name: Set up Python
24+
uses: actions/setup-python@v4
25+
with:
26+
python-version: '3.x'
27+
28+
- name: Compute Kani Metrics
29+
run: ./scripts/run-kani.sh --run metrics --path ${{github.workspace}}
30+
31+
- name: Create Pull Request
32+
uses: peter-evans/create-pull-request@v7
33+
with:
34+
commit-message: Update Kani metrics
35+
title: 'Update Kani Metrics'
36+
body: |
37+
This is an automated PR to update Kani metrics.
38+
39+
The metrics have been updated by running `./scripts/run-kani.sh --run metrics`.
40+
branch: update-kani-metrics
41+
delete-branch: true
42+
base: main

.gitignore

+2-1
Original file line numberDiff line numberDiff line change
@@ -26,6 +26,8 @@ library/target
2626
*.rmeta
2727
*.mir
2828
Cargo.lock
29+
/doc/mdbook-metrics/target
30+
*.png
2931

3032
## Temporary files
3133
*~
@@ -46,7 +48,6 @@ package-lock.json
4648
## Kani
4749
*.out
4850

49-
5051
# Added by cargo
5152
#
5253
# already existing elements were commented out

doc/book.toml

+6
Original file line numberDiff line numberDiff line change
@@ -19,5 +19,11 @@ runnable = false
1919

2020
[output.linkcheck]
2121

22+
[preprocessor.metrics]
23+
# Note that the manifest-path is doc/mdbook-metrics, meaning that to build this book, you need to run "mdbook build doc"
24+
# rather than "mdbook build" from inside the doc/ directory.
25+
# We choose the former because our "Build Book" Github workflow runs "mdbook build doc."
26+
command = "cargo run --manifest-path=doc/mdbook-metrics/Cargo.toml"
27+
2228
[rust]
2329
edition = "2021"

doc/mdbook-metrics/Cargo.toml

+8
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,8 @@
1+
[package]
2+
name = "mdbook-kani-metrics"
3+
version = "0.1.0"
4+
edition = "2021"
5+
6+
[dependencies]
7+
mdbook = { version = "^0.4" }
8+
serde_json = "1.0.132"

doc/mdbook-metrics/src/main.rs

+106
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,106 @@
1+
use mdbook::book::{Book, Chapter};
2+
use mdbook::errors::Error;
3+
use mdbook::preprocess::{CmdPreprocessor, Preprocessor, PreprocessorContext};
4+
use mdbook::BookItem;
5+
use std::{env, io, path::Path, process::Command};
6+
7+
fn main() {
8+
let mut args = std::env::args().skip(1);
9+
match args.next().as_deref() {
10+
Some("supports") => {
11+
// Supports all renderers.
12+
return;
13+
}
14+
Some(arg) => {
15+
eprintln!("unknown argument: {arg}");
16+
std::process::exit(1);
17+
}
18+
None => {}
19+
}
20+
21+
if let Err(e) = handle_preprocessing() {
22+
eprintln!("{}", e);
23+
std::process::exit(1);
24+
}
25+
}
26+
27+
// Plot the Kani metrics.
28+
// The mdbook builder reads the postprocessed book from stdout,
29+
// so suppress all Command output to avoid having its output interpreted as part of the book
30+
fn run_kani_metrics_script() -> Result<(), Error> {
31+
// This will be the absolute path to the "doc/" folder
32+
let original_dir = env::current_dir()?;
33+
let tools_path = original_dir.join(Path::new("doc/src/tools"));
34+
35+
let kani_std_analysis_path = Path::new("scripts/kani-std-analysis");
36+
env::set_current_dir(kani_std_analysis_path)?;
37+
38+
Command::new("pip")
39+
.args(&["install", "-r", "requirements.txt"])
40+
.stdout(std::process::Stdio::null())
41+
.stderr(std::process::Stdio::null())
42+
.status()?;
43+
44+
Command::new("python")
45+
.args(&[
46+
"kani_std_analysis.py",
47+
"--plot-only",
48+
"--plot-dir",
49+
&tools_path.to_string_lossy(),
50+
])
51+
.stdout(std::process::Stdio::null())
52+
.stderr(std::process::Stdio::null())
53+
.status()?;
54+
55+
env::set_current_dir(&original_dir)?;
56+
57+
Ok(())
58+
}
59+
60+
struct AddKaniGraphs;
61+
62+
impl Preprocessor for AddKaniGraphs {
63+
fn name(&self) -> &str {
64+
"add-metrics"
65+
}
66+
67+
fn run(&self, _ctx: &PreprocessorContext, mut book: Book) -> Result<Book, Error> {
68+
book.for_each_mut(|item| {
69+
if let BookItem::Chapter(ch) = item {
70+
if ch.name == "Kani" {
71+
add_graphs(ch);
72+
return;
73+
}
74+
}
75+
});
76+
Ok(book)
77+
}
78+
}
79+
80+
fn add_graphs(chapter: &mut Chapter) {
81+
let new_content = r#"
82+
## Kani Metrics
83+
84+
Note that these metrics are for x86-64 architectures.
85+
86+
## `core`
87+
![Unsafe Metrics](core_unsafe_metrics.png)
88+
89+
![Safe Abstractions Metrics](core_safe_abstractions_metrics.png)
90+
91+
![Safe Metrics](core_safe_metrics.png)
92+
"#;
93+
94+
chapter.content.push_str(new_content);
95+
}
96+
97+
pub fn handle_preprocessing() -> Result<(), Error> {
98+
run_kani_metrics_script()?;
99+
let pre = AddKaniGraphs;
100+
let (ctx, book) = CmdPreprocessor::parse_input(io::stdin())?;
101+
102+
let processed_book = pre.run(&ctx, book)?;
103+
serde_json::to_writer(io::stdout(), &processed_book)?;
104+
105+
Ok(())
106+
}

doc/src/SUMMARY.md

-1
Original file line numberDiff line numberDiff line change
@@ -9,7 +9,6 @@
99
- [Verification Tools](./tools.md)
1010
- [Kani](./tools/kani.md)
1111

12-
1312
---
1413

1514
- [Challenges](./challenges.md)

0 commit comments

Comments
 (0)