Auto merge of #812 - fitzgen:dump-preprocessed, r=emilio

Dump preprocessed input headers

See each commit message.

Fixes #811.

r? @emilio

Author: bors-servo

.gitignore

@@ -8,3 +8,6 @@ tests/expectations/Cargo.lock
 # Test script output
 ir.dot
 ir.png
+
+# Output of the --dump-preprocessed-input flag.
+__bindgen.*

CONTRIBUTING.md

@@ -272,14 +272,21 @@ With those two things in hand, running `creduce` looks like this:
 
 ### Isolating Your Test Case
 
-Use the `-save-temps` flag to make Clang spit out its intermediate
-representations when compiling the test case into an object file.
+If you're using `bindgen` as a command line tool, pass
+`--dump-preprocessed-input` flag.
 
-    $ clang[++ -x c++ --std=c++14] -save-temps -c my_test_case.h
+If you're using `bindgen` as a Rust library, invoke the
+`bindgen::Builder::dump_preprocessed_input` method where you call
+`bindgen::Builder::generate`.
 
-There should now be a `my_test_case.ii` file, which is the results after the C
-pre-processor has processed all the `#include`s, `#define`s, and `#ifdef`s. This
-is generally what we're looking for.
+Afterwards, there should be a `__bindgen.i` or `__bindgen.ii` file containing
+the combined and preprocessed input headers, which is usable as an isolated,
+standalone test case.
+
+Note that the preprocessor likely removed all comments, so if the bug you're
+trying to pin down involves source annotations (for example, `/** <div
+rustbindgen opaque> */`), then you will have to manually reapply them to the
+preprocessed file.
 
 ### Writing a Predicate Script
 

src/lib.rs

@@ -87,9 +87,11 @@ use ir::item::Item;
 use parse::{ClangItemParser, ParseError};
 use regex_set::RegexSet;
 
-use std::fs::OpenOptions;
+use std::fs::{File, OpenOptions};
+use std::iter;
 use std::io::{self, Write};
-use std::path::Path;
+use std::path::{Path, PathBuf};
+use std::process::Command;
 use std::sync::Arc;
 
 use syntax::ast;
@@ -165,9 +167,12 @@ impl Default for CodegenConfig {
 /// // Write the generated bindings to an output file.
 /// try!(bindings.write_to_file("path/to/output.rs"));
 /// ```
-#[derive(Debug,Default)]
+#[derive(Debug, Default)]
 pub struct Builder {
     options: BindgenOptions,
+    input_headers: Vec<String>,
+    // Tuples of unsaved file contents of the form (name, contents).
+    input_header_contents: Vec<(String, String)>,
 }
 
 /// Construct a new [`Builder`](./struct.Builder.html).
@@ -180,9 +185,9 @@ impl Builder {
     pub fn command_line_flags(&self) -> Vec<String> {
         let mut output_vector: Vec<String> = Vec::new();
 
-        if let Some(ref header) = self.options.input_header {
-            //Positional argument 'header'
-            output_vector.push(header.clone().into());
+        if let Some(header) = self.input_headers.last().cloned() {
+            // Positional argument 'header'
+            output_vector.push(header);
         }
 
         self.options
@@ -412,16 +417,19 @@ impl Builder {
                  })
             .count();
 
+        output_vector.push("--".into());
+
         if !self.options.clang_args.is_empty() {
-            output_vector.push("--".into());
-            self.options
-                .clang_args
-                .iter()
-                .cloned()
-                .map(|item| {
-                    output_vector.push(item);
-                })
-                .count();
+            output_vector.extend(
+                self.options
+                    .clang_args
+                    .iter()
+                    .cloned()
+            );
+        }
+
+        if self.input_headers.len() > 1 {
+            output_vector.extend(self.input_headers[..self.input_headers.len() - 1].iter().cloned());
         }
 
         output_vector
@@ -450,21 +458,15 @@ impl Builder {
     ///     .unwrap();
     /// ```
     pub fn header<T: Into<String>>(mut self, header: T) -> Builder {
-        if let Some(prev_header) = self.options.input_header.take() {
-            self.options.clang_args.push("-include".into());
-            self.options.clang_args.push(prev_header);
-        }
-
-        let header = header.into();
-        self.options.input_header = Some(header);
+        self.input_headers.push(header.into());
         self
     }
 
     /// Add `contents` as an input C/C++ header named `name`.
     ///
     /// The file `name` will be added to the clang arguments.
     pub fn header_contents(mut self, name: &str, contents: &str) -> Builder {
-        self.options.input_unsaved_files.push(clang::UnsavedFile::new(name, contents));
+        self.input_header_contents.push((name.into(), contents.into()));
         self
     }
 
@@ -794,9 +796,94 @@ impl Builder {
     }
 
     /// Generate the Rust bindings using the options built up thus far.
-    pub fn generate<'ctx>(self) -> Result<Bindings<'ctx>, ()> {
+    pub fn generate<'ctx>(mut self) -> Result<Bindings<'ctx>, ()> {
+        self.options.input_header = self.input_headers.pop();
+        self.options.clang_args.extend(
+            self.input_headers
+                .drain(..)
+                .flat_map(|header| {
+                    iter::once("-include".into())
+                        .chain(iter::once(header))
+                })
+        );
+
+        self.options.input_unsaved_files.extend(
+            self.input_header_contents
+                .drain(..)
+                .map(|(name, contents)| clang::UnsavedFile::new(&name, &contents))
+        );
+
         Bindings::generate(self.options, None)
     }
+
+    /// Preprocess and dump the input header files to disk.
+    ///
+    /// This is useful when debugging bindgen, using C-Reduce, or when filing
+    /// issues. The resulting file will be named something like `__bindgen.i` or
+    /// `__bindgen.ii`
+    pub fn dump_preprocessed_input(&self) -> io::Result<()> {
+        let clang = clang_sys::support::Clang::find(None, &[])
+            .ok_or_else(|| io::Error::new(io::ErrorKind::Other,
+                                          "Cannot find clang executable"))?;
+
+        // The contents of a wrapper file that includes all the input header
+        // files.
+        let mut wrapper_contents = String::new();
+
+        // Whether we are working with C or C++ inputs.
+        let mut is_cpp = false;
+
+        // For each input header, add `#include "$header"`.
+        for header in &self.input_headers {
+            is_cpp |= header.ends_with(".hpp");
+
+            wrapper_contents.push_str("#include \"");
+            wrapper_contents.push_str(header);
+            wrapper_contents.push_str("\"\n");
+        }
+
+        // For each input header content, add a prefix line of `#line 0 "$name"`
+        // followed by the contents.
+        for &(ref name, ref contents) in &self.input_header_contents {
+            is_cpp |= name.ends_with(".hpp");
+
+            wrapper_contents.push_str("#line 0 \"");
+            wrapper_contents.push_str(name);
+            wrapper_contents.push_str("\"\n");
+            wrapper_contents.push_str(contents);
+        }
+
+        is_cpp |= self.options.clang_args.windows(2).any(|w| {
+            w[0] == "-x=c++" || w[1] == "-x=c++" || w == &["-x", "c++"]
+        });
+
+        let wrapper_path = PathBuf::from(if is_cpp {
+            "__bindgen.cpp"
+        } else {
+            "__bindgen.c"
+        });
+
+        {
+            let mut wrapper_file = File::create(&wrapper_path)?;
+            wrapper_file.write(wrapper_contents.as_bytes())?;
+        }
+
+        let mut cmd = Command::new(&clang.path);
+        cmd.arg("-save-temps")
+            .arg("-c")
+            .arg(&wrapper_path);
+
+        for a in &self.options.clang_args {
+            cmd.arg(a);
+        }
+
+        if cmd.spawn()?.wait()?.success() {
+            Ok(())
+        } else {
+            Err(io::Error::new(io::ErrorKind::Other,
+                               "clang exited with non-zero status"))
+        }
+    }
 }
 
 /// Configuration options for generated bindings.

src/options.rs

@@ -214,6 +214,12 @@ pub fn builder_from_flags<I>
             Arg::with_name("verbose")
                 .long("verbose")
                 .help("Print verbose error messages."),
+            Arg::with_name("dump-preprocessed-input")
+                .long("dump-preprocessed-input")
+                .help("Preprocess and dump the input header files to disk. \
+                       Useful when debugging bindgen, using C-Reduce, or when \
+                       filing issues. The resulting file will be named \
+                       something like `__bindgen.i` or `__bindgen.ii`.")
         ]) // .args()
         .get_matches_from(args);
 
@@ -424,6 +430,10 @@ pub fn builder_from_flags<I>
         Box::new(io::BufWriter::new(io::stdout())) as Box<io::Write>
     };
 
+    if matches.is_present("dump-preprocessed-input") {
+        builder.dump_preprocessed_input()?;
+    }
+
     let verbose = matches.is_present("verbose");
 
     Ok((builder, output, verbose))

tests/tests.rs

@@ -238,3 +238,31 @@ fn no_system_header_includes() {
             .expect("should wait for ./ci/no-includes OK")
             .success());
 }
+
+#[test]
+fn dump_preprocessed_input() {
+    let arg_keyword = concat!(env!("CARGO_MANIFEST_DIR"), "/tests/headers/arg_keyword.hpp");
+    let empty_layout = concat!(env!("CARGO_MANIFEST_DIR"), "/tests/headers/cpp-empty-layout.hpp");
+
+    builder()
+        .header(arg_keyword)
+        .header(empty_layout)
+        .dump_preprocessed_input()
+        .expect("should dump preprocessed input");
+
+    fn slurp(p: &str) -> String {
+        let mut contents = String::new();
+        let mut file = fs::File::open(p).unwrap();
+        file.read_to_string(&mut contents).unwrap();
+        contents
+    }
+
+    let bindgen_ii = slurp("__bindgen.ii");
+    let arg_keyword = slurp(arg_keyword);
+    let empty_layout = slurp(empty_layout);
+
+    assert!(bindgen_ii.find(&arg_keyword).is_some(),
+            "arg_keyword.hpp is in the preprocessed file");
+    assert!(bindgen_ii.find(&empty_layout).is_some(),
+            "cpp-empty-layout.hpp is in the preprocessed file");
+}