Detect which reference examples should check/codegen/verify. (rust-lang#363)

bdalrhm · tedinski · commit c550d9531165 · 2021-07-28T16:49:42.000-04:00
* Copy options from the rust code blocks in the markdown files.

* Fix typos and add comments.
diff --git a/scripts/rmc.py b/scripts/rmc.py
@@ -142,6 +142,8 @@ def compile_single_rust_file(input_filename, output_filename, verbose=False, deb
                  "-Z", f"symbol-mangling-version={mangler}", 
                  "-Z", f"symbol_table_passes={' '.join(symbol_table_passes)}",
                  f"--cfg={RMC_CFG}", "-o", output_filename, input_filename]
+    if "RUSTFLAGS" in os.environ:
+        build_cmd += os.environ["RUSTFLAGS"].split(" ")
     build_env = os.environ
     if debug:
         add_rmc_rustc_debug_to_env(build_env)
diff --git a/src/tools/compiletest/src/runtest.rs b/src/tools/compiletest/src/runtest.rs
@@ -2394,12 +2394,12 @@ impl<'test> TestCx<'test> {
 
     /// Adds rmc scripts directory to the `PATH` environment variable.
     fn add_rmc_dir_to_path(&self, command: &mut Command) {
-        // If the PATH enviornment variable is already defined,
+        // If the PATH environment variable is already defined,
         if let Some((key, val)) = env::vars().find(|(key, _)| key == "PATH") {
             // Add the RMC scripts directory to the PATH.
             command.env(key, format!("{}:{}", self.config.rmc_dir_path.to_str().unwrap(), val));
         } else {
-            // Otherwise, insert PATH as a new enviornment variable and set its value to the RMC scripts directory.
+            // Otherwise, insert PATH as a new environment variable and set its value to the RMC scripts directory.
             command.env(
                 String::from("PATH"),
                 String::from(self.config.rmc_dir_path.to_str().unwrap()),
@@ -2411,7 +2411,10 @@ impl<'test> TestCx<'test> {
     /// error message is printed to stdout if the check result is not expected.
     fn check(&self) {
         let mut rustc = Command::new("rmc-rustc");
-        rustc.args(["-Z", "no-codegen"]).arg(&self.testpaths.file);
+        rustc
+            .args(self.props.compile_flags.clone())
+            .args(["-Z", "no-codegen"])
+            .arg(&self.testpaths.file);
         self.add_rmc_dir_to_path(&mut rustc);
         let proc_res = self.compose_and_run_compiler(rustc, None);
         if self.props.rmc_panic_step == Some(RMCFailStep::Check) {
@@ -2431,6 +2434,7 @@ impl<'test> TestCx<'test> {
     fn codegen(&self) {
         let mut rustc = Command::new("rmc-rustc");
         rustc
+            .args(self.props.compile_flags.clone())
             .args(["-Z", "codegen-backend=gotoc", "--cfg=rmc", "--out-dir"])
             .arg(self.output_base_dir())
             .arg(&self.testpaths.file);
@@ -2461,6 +2465,13 @@ impl<'test> TestCx<'test> {
         // 2. It may pass some options that do not make sense for RMC
         // So we create our own command to execute RMC and pass it to self.compose_and_run_compiler(...) directly.
         let mut rmc = Command::new("rmc");
+        // We cannot pass rustc flags directly to RMC. Instead, we add them
+        // to the current environment through the `RUSTFLAGS` environment
+        // variable. RMC recognizes the variable and adds those flags to its
+        // internal call to rustc.
+        if !self.props.compile_flags.is_empty() {
+            rmc.env("RUSTFLAGS", self.props.compile_flags.join(" "));
+        }
         // Pass the test path along with RMC and CBMC flags parsed from comments at the top of the test file.
         rmc.args(&self.props.rmc_flags)
             .arg("--input")
diff --git a/src/tools/dashboard/src/reference.rs b/src/tools/dashboard/src/reference.rs
@@ -8,7 +8,10 @@ use crate::dashboard;
 use pulldown_cmark::{Parser, Tag};
 use std::{
     collections::HashMap,
-    env, fs,
+    env,
+    fmt::{Debug, Formatter, Result},
+    fs::{self, File},
+    hash::Hash,
     io::{BufRead, BufReader},
     path::{Path, PathBuf},
     process::{Command, Stdio},
@@ -19,17 +22,18 @@ use std::{
 /// code to corresponding directories where the extracted rust code should
 /// reside.
 fn parse_hierarchy(summary_path: &Path) -> HashMap<PathBuf, PathBuf> {
+    let summary_dir = summary_path.parent().unwrap().to_path_buf();
     let start = "# The Rust Reference\n\n[Introduction](introduction.md)";
     let summary = fs::read_to_string(summary_path).unwrap();
     assert!(summary.starts_with(start), "Error: The start of the summary file changed.");
     // Skip the title and introduction.
     let n = Parser::new(start).count();
     let parser = Parser::new(&summary).skip(n);
     // Set "ref" as the root of the hierarchical path.
-    let mut hierarchy = PathBuf::from("ref");
+    let mut hierarchy: PathBuf = ["src", "test", "ref"].iter().collect();
     let mut map = HashMap::new();
     // Introduction is a especial case, so handle it separately.
-    map.insert(PathBuf::from("introduction.md"), hierarchy.join("Introduction"));
+    map.insert(summary_dir.join("introduction.md"), hierarchy.join("Introduction"));
     for event in parser {
         match event {
             pulldown_cmark::Event::End(Tag::Item) => {
@@ -42,7 +46,9 @@ fn parse_hierarchy(summary_path: &Path) -> HashMap<PathBuf, PathBuf> {
                 // contain the title of the current chapter/section. So, we wait
                 // for the end of the link tag before adding the path and
                 // hierarchy of the current chapter/section to the map.
-                map.insert(path.split('/').collect(), hierarchy.clone());
+                let mut full_path = summary_dir.clone();
+                full_path.extend(path.split('/'));
+                map.insert(full_path, hierarchy.clone());
             }
             pulldown_cmark::Event::Text(text) => {
                 // Add the current chapter/section title to the hierarchy.
@@ -54,87 +60,131 @@ fn parse_hierarchy(summary_path: &Path) -> HashMap<PathBuf, PathBuf> {
     map
 }
 
-/// Extracts examples from the given relative `paths` in the `book_dir` and
-/// saves them in `gen_dir`.
-fn extract_examples(paths: Vec<&PathBuf>, book_dir: &Path, gen_dir: &Path) {
-    for path in paths {
-        let mut cmd = Command::new("rustdoc");
-        cmd.args([
-            "+nightly",
-            "--test",
-            "-Z",
-            "unstable-options",
-            book_dir.join(path).to_str().unwrap(),
-            "--test-builder",
-            &["src", "tools", "dashboard", "print.sh"]
-                .iter()
-                .collect::<PathBuf>()
-                .to_str()
-                .unwrap(),
-            "--persist-doctests",
-            gen_dir.to_str().unwrap(),
-            "--no-run",
-        ]);
-        cmd.stdout(Stdio::null());
-        cmd.spawn().unwrap().wait().unwrap();
+/// The data structure represents the "full" path to examples in the Rust books.
+#[derive(PartialEq, Eq, Hash)]
+struct Example {
+    /// Path to the markdown file containing the example.
+    path: PathBuf,
+    /// Line number of the code block introducing the example.
+    line: usize,
+}
+
+impl Example {
+    /// Creates a new [`Example`] instance representing "full" path to the
+    /// Rust example.
+    fn new(path: PathBuf, line: usize) -> Example {
+        Example { path, line }
     }
 }
 
-/// Copies the extracted rust code in `from_dir` to `src/test` following the
-/// hierarchy specified by `map`.
-fn organize_examples(map: &HashMap<PathBuf, PathBuf>, book_dir: &Path, from_dir: &Path) {
-    // The names of the extracted examples generated by `rustdoc` have the
-    // format `<path>_<line-num>_<test-num>` where occurrences of '/', '-', and
-    // '.' in <path> are replaced by '_'. This transformation is not injective,
-    // so we cannot map those names back to the original markdown file path.
-    // Instead, we apply the same transformation on the keys of `map` in the for
-    // loop below and lookup <path> in those modified keys.
-    let mut modified_map = HashMap::new();
-    for (path, hierarchy) in map.iter() {
-        modified_map.insert(
-            book_dir.join(path).to_str().unwrap().replace(&['\\', '/', '-', '.'][..], "_"),
-            hierarchy.clone(),
-        );
+impl Debug for Example {
+    fn fmt(&self, f: &mut Formatter<'_>) -> Result {
+        f.write_fmt(format_args!("{}:{}", self.path.to_str().unwrap(), self.line))
     }
-    for dir in from_dir.read_dir().unwrap() {
-        let dir = dir.unwrap().path();
+}
+
+/// Extracts examples from the markdown files specified by each key in the given
+/// `map` and saves them in the directory specified by the corresponding value.
+/// Returns a mapping from the original location of **_each_** example to the
+/// path it was extracted to.
+fn extract_examples(par_map: HashMap<PathBuf, PathBuf>) -> HashMap<Example, PathBuf> {
+    let mut full_map = HashMap::new();
+    for (par_from, par_to) in par_map {
+        let pairs = extract(&par_from, &par_to);
+        for (key, val) in pairs {
+            full_map.insert(key, val);
+        }
+    }
+    full_map
+}
+
+/// Extracts examples from the markdown files specified by `par_from` and saves
+/// them in the directory specified by `par_to`. Returns a mapping from the
+/// original location of **_each_** example to the path it was extracted to.
+fn extract(par_from: &Path, par_to: &Path) -> Vec<(Example, PathBuf)> {
+    let build_dir = &env::var("BUILD_DIR").unwrap();
+    let triple = &env::var("TRIPLE").unwrap();
+    // Create a temporary directory to save the files generated by `rustdoc`.
+    let gen_dir: PathBuf = [build_dir, triple, "dashboard", "ref"].iter().collect();
+    fs::create_dir_all(&gen_dir).unwrap();
+    let mut cmd = Command::new("rustdoc");
+    cmd.args([
+        "+nightly",
+        "--test",
+        "-Z",
+        "unstable-options",
+        par_from.to_str().unwrap(),
+        "--test-builder",
+        &["src", "tools", "dashboard", "print.sh"].iter().collect::<PathBuf>().to_str().unwrap(),
+        "--persist-doctests",
+        gen_dir.to_str().unwrap(),
+        "--no-run",
+    ]);
+    cmd.stdout(Stdio::null());
+    cmd.spawn().unwrap().wait().unwrap();
+    // Mapping from path and line number of rust example to where it was extracted to.
+    let mut pairs = Vec::new();
+
+    for dir in gen_dir.read_dir().unwrap() {
         // Some directories do not contain tests because the markdown file
-        // instructs `rustdoc` to ignore those tests.
-        if let Some(example) = dir.read_dir().unwrap().next() {
-            let example = example.unwrap().path();
-            copy(&example, &modified_map);
+        // instructs `rustdoc` to "ignore" those tests.
+        let dir = dir.unwrap().path();
+        if let Some(from) = dir.read_dir().unwrap().next() {
+            // The path to each example extracted by `rustdoc` has the form:
+            // <from> = `<gen_dir>/<par_from>_<line>_<test-num>/rust_out`
+            // where occurrences of '/', '-', and '.' in <par_from> are replaced
+            // by '_'. We copy the file in this path to a new path of the form:
+            // <to> = `<par_to>/<line>.rs`
+            //  We omit <test-num> because all tests have the same number, 0.
+            let from = from.unwrap().path();
+            let path_line_test = dir.file_name().unwrap().to_str().unwrap();
+            let splits: Vec<_> = path_line_test.rsplitn(3, '_').collect();
+            let line: usize = splits[1].parse().unwrap();
+            let to = par_to.join(format!("{}.rs", line));
+            fs::create_dir_all(par_to).unwrap();
+            fs::copy(&from, &to).unwrap();
+            pairs.push((Example::new(par_from.to_path_buf(), line), to));
         }
     }
+    // Delete the temporary directory.
+    fs::remove_dir_all(gen_dir).unwrap();
+    pairs
 }
 
-/// Copy the file specified by `from` to the corresponding location specified by
-/// `map`.
-fn copy(from: &Path, map: &HashMap<String, PathBuf>) {
-    // The path specified by `from` has the form:
-    // `build/<triple>/dashboard/ref/<key>_<line-num>_<test-num>/rust_out`
-    // We copy the file in this path to a new path of the form:
-    // `src/test/<val>/<line-num>.rs
-    // where `map[<key>] == <val>`. We omit <test-num> because all tests have
-    // the same number, 0.
-    // Extract `<key>_<line-num>_<test-num>`.
-    let key_line_test = from.parent().unwrap().file_name().unwrap().to_str().unwrap();
-    // Extract <key> and <line-num> from `key_line_test` to get <val> and
-    // construct destination path.
-    let splits: Vec<_> = key_line_test.rsplitn(3, '_').collect();
-    let key = splits[2];
-    let line = splits[1];
-    let val = &map[key];
-    let name = &format!("{}.rs", line);
-    let to = Path::new("src").join("test").join(val).join(name);
-    fs::create_dir_all(to.parent().unwrap()).unwrap();
-    fs::copy(&from, &to).unwrap();
+/// Prepends the text in `path` with the given `text`.
+fn prepend_text(path: &Path, text: &str) {
+    let code = fs::read_to_string(&path).unwrap();
+    let code = format!("{}\n{}", text, code);
+    fs::write(&path, code).unwrap();
 }
 
-/// Pre-processes the tests in the specified `paths` before running them with
-/// `compiletest`.
-fn preprocess_examples(_paths: Vec<&PathBuf>) {
-    // For now, we will only pre-process the tests that cause infinite loops.
-    // TODO: properly implement this step (see issue #324).
+/// Pre-processes the examples in `map` before running them with `compiletest`.
+fn preprocess_examples(map: &HashMap<Example, PathBuf>) {
+    // Copy compiler configurations specified in the original markdown code
+    // block.
+    for (from, to) in map.iter() {
+        let file = File::open(&from.path).unwrap();
+        // Skip to the first line of the example code block.
+        // Line numbers in files start with 1 but `nth(...)` starts with 0.
+        // Subtract 1 to account for the difference.
+        let line = BufReader::new(file).lines().nth(from.line - 1).unwrap().unwrap();
+        if line.contains("edition2015") {
+            prepend_text(to, "// compile-flags: --edition 2015");
+        } else {
+            prepend_text(to, "// compile-flags: --edition 2018");
+        }
+        // Most examples with `compile_fail` configuration fail because of
+        // check errors.
+        if line.contains("compile_fail") {
+            prepend_text(to, "// rmc-check-fail");
+        }
+        // RMC should catch run-time errors.
+        if line.contains("should_panic") {
+            prepend_text(to, "// rmc-verify-fail");
+        }
+    }
+    // For now, we will only manually pre-process the tests that cause infinite loops.
+    // TODO: Add support for manually adding options and assertions (see issue #324).
     let loop_tests: [PathBuf; 4] = [
         ["src", "test", "ref", "Appendices", "Glossary", "263.rs"].iter().collect(),
         ["src", "test", "ref", "Linkage", "190.rs"].iter().collect(),
@@ -193,7 +243,6 @@ fn run_examples(suite: &str, log_path: &Path) {
     ]);
     cmd.env_clear().envs(filtered_env);
     cmd.stdout(Stdio::null());
-
     cmd.spawn().unwrap().wait().unwrap();
 }
 
@@ -258,20 +307,17 @@ fn display_dashboard(dashboard: dashboard::Tree) {
 /// displays their results in a terminal dashboard.
 pub fn display_reference_dashboard() {
     let summary_path: PathBuf = ["src", "doc", "reference", "src", "SUMMARY.md"].iter().collect();
-    let ref_dir: PathBuf = ["src", "doc", "reference", "src"].iter().collect();
     let build_dir = &env::var("BUILD_DIR").unwrap();
     let triple = &env::var("TRIPLE").unwrap();
-    let gen_dir: PathBuf = [build_dir, triple, "dashboard", "ref"].iter().collect();
     let log_path: PathBuf = [build_dir, triple, "dashboard", "ref.log"].iter().collect();
     // Parse the chapter/section hierarchy from the table of contents in The
     // Rust Reference.
     let map = parse_hierarchy(&summary_path);
-    // Extract examples from The Rust Reference.
-    extract_examples(map.keys().collect(), &ref_dir, &gen_dir);
-    // Reorganize those examples following the The Rust Reference hierarchy.
-    organize_examples(&map, &ref_dir, &gen_dir);
+    // Extract examples from The Rust Reference, organize them following the
+    // partial hierarchy in map, and return the full hierarchy map.
+    let map = extract_examples(map);
     // Pre-process the examples before running them through `compiletest`.
-    preprocess_examples(map.values().collect());
+    preprocess_examples(&map);
     // Run `compiletest` on the reference examples.
     run_examples("ref", &log_path);
     // Parse `compiletest` log file.
